module RoME

using Reexport

@reexport using IncrementalInference
@reexport using TransformUtils
@reexport using Distributions
@reexport using KernelDensityEstimate
@reexport using ApproxManifoldProducts

using
  Dates,
  FileIO,
  Distributed,
  LinearAlgebra,
  Statistics,
  Rotations,
  CoordinateTransformations,
  JLD2,
  ProgressMeter,
  DocStringExtensions,
  DistributedFactorGraphs,
  TensorCast,
  ManifoldsBase

using StaticArrays

# to avoid name conflicts
import Manifolds
using Manifolds: hat, ProductGroup, ProductManifold, SpecialEuclidean, ProductRepr, SpecialOrthogonal, TranslationGroup, identity_element

import Manifolds: project, project!

export SpecialOrthogonal, SpecialEuclidean

# using Graphs,  # TODO determine how many parts still require Graphs still directly

import Base: +, \, convert
import TransformUtils: ⊖, ⊕, convert, ominus, veeQuaternion
import IncrementalInference: MB
import IncrementalInference: convert, getSample, reshapeVec2Mat, DFG
# not sure why this is gives import error
import DistributedFactorGraphs: compare
import DistributedFactorGraphs: getDimension, getManifold

# const AMP = ApproxManifoldProducts

const InstanceType{T} = Union{Type{<:T},T}

export
  IIF,  # Aliases for various packages
  KDE,
  TU,
  AMP,
  DFG,

  # RoME specific functions
  measureMeanDist,
  predictBodyBR,
  calcPosePointBearingRange,
  getLastPose,
  getLastPose2D,
  odomKDE,
  initFactorGraph!,
  addOdoFG!,
  addposeFG!,
  newLandm!,
  addBRFG!,
  addMMBRFG!,
  addAutoLandmBR!,
  projNewLandm!,
  malahanobisBR,
  \,
  RangeAzimuthElevation,

  # helper functions
  get2DSamples,
  get2DSampleMeans,
  getAll2DMeans,
  getAll2DPoses,
  get2DPoseSamples,
  get2DPoseMeans,
  get2DPoseMax,
  get2DLandmSamples,
  get2DLandmMeans,
  get2DLandmMax,

  # helper functions
  getLastLandm2D,
  getLastPose2D,
  getNextLbl,

  # RobotUtils
  getRangeKDEMax2D,
  getLastPoses,
  setSolvableOldPoses!,

  # some transform functions
  cart2pol,
  pol2cart,

  # Feature tracking code
  Feature,
  initTrackersFrom,
  propAllTrackers!,
  measUpdateTrackers!,
  assocMeasWFeats!,

  lsrBR,

  # Didson model
  evalPotential,
  LinearRangeBearingElevation,
  project!,
  project,
  # backprojectRandomized!,
  residual!,
  residualLRBE!,
  reuseLBRA,
  ominus,
  ominus!,
  +,
  evalPotential,
  getSample!,
  getSample,

  # Didson convenience function
  addLinearArrayConstraint,

  # camera model -- TODO --separate out
  CameraIntrinsic,
  CameraExtrinsic,
  CameraModelFull,
  project!,
  project,
  # keep
  cameraResidual!,

  # Point2D
  Point2,
  Point2Point2,
  PackedPoint2Point2,
  Point2Point2Range,
  PackedPoint2Point2Range,
  PriorPoint2,
  PackedPriorPoint2,
  Pose2Point2BearingRange,
  Pose2Point2BearingRangeMH,
  PackedPose2Point2BearingRange,
  PackedPose2Point2BearingRangeMH,
  Pose2Point2Bearing,
  PackedPose2Point2Bearing,
  Pose2Point2Range,
  PackedPose2Point2Range,
  PriorPoint2,
  PackedPriorPoint2,

  # Velocity in Point2 types
  DynPoint2,
  DynPoint2VelocityPrior,
  DynPoint2DynPoint2,
  VelPoint2VelPoint2,
  Point2Point2Velocity,
  PackedDynPoint2VelocityPrior,
  PackedVelPoint2VelPoint2,

  Point3Point3,
  PackedPoint3Point3,

  # likely to be deprecated
  solveLandm,
  solvePose2,
  solveSetSeps,
  addPose2Pose2!,
  compareDensity,

  # Pose2D
  Pose2,
  PriorPose2,
  PackedPriorPose2,
  PartialPriorYawPose2,
  PackedPartialPriorYawPose2,
  Pose2Pose2,
  PackedPose2Pose2,
  # Will be deprecated
  addPose2Pose2,

  # Polar types
  Polar,
  PolarPolar,
  PriorPolar,

  # MultipleFeatures2D constraint functions
  MultipleFeatures2D,
  getUvecScaleFeature2D,
  getUvecScaleBaseline2D,

  # Pose3, Three dimensional
  Pose3,
  Point3,
  PriorPoint3,

  # partial Pose3
  PriorPose3ZRP,
  Pose3Pose3XYYaw,
  PackedPose3Pose3XYYaw,
  PackedPriorPose3ZRP,
  # Various utilities
  passTypeThrough,

  # SLAM specific functions
  SLAMWrapper,

  # FG Analysis tools
  rangeErrMaxPoint2,
  rangeCompAllPoses,
  rangeCompAllPoses,

  # new robot navigation functionality
  triggerPose,
  GenericInSituSystem,
  InSituSystem,
  makeInSituSys,
  makeGenericInSituSys,
  advOdoByRules,
  poseTrigAndAdd!,
  poseTrigAndAdd!,
  processTreeTrackersUpdates!,
  addSoftEqualityPoint2D,
  vectoarr2,
  basicFactorGraphExample,
  predictVariableByFactor,

  # jld required Features Type
  LaserFeatures

include("SpecialDefinitions.jl")

#uses DFG v0.10.2 @defVariable for above
include("variables/Local_Manifold_Workaround.jl")
include("variables/VariableTypes.jl")

## More factor types
# RoME internal factors (FYI outside factors are easy, see Caesar documentation)
include("factors/Point2D.jl")
include("factors/Range2D.jl")
include("factors/Bearing2D.jl")
include("factors/BearingRange2D.jl")
include("factors/Polar.jl")
include("factors/PriorPose2.jl")
include("factors/PartialPriorPose2.jl")
include("factors/Pose2D.jl")
include("factors/Pose2Point2.jl")
include("factors/MutablePose2Pose2.jl")
include("factors/DynPoint2D.jl")
include("factors/VelPoint2D.jl")
include("factors/DynPose2D.jl")
include("factors/VelPose2D.jl")
include("factors/Point3D.jl")
include("factors/Point3Point3.jl")
include("factors/Pose3D.jl")
include("factors/Pose3Pose3.jl")
include("factors/PartialPose3.jl")
include("factors/MultipleFeaturesConstraint.jl")
include("factors/InertialPose3.jl")

# tools that come and go
include("TemporaryFunctionality.jl")

# additional tools
include("FactorGraphAnalysisTools.jl")

# tools related to robotics
include("BayesTracker.jl")
include("SensorModels.jl")
include("CameraModel.jl")
include("Slam.jl")
include("RobotUtils.jl")
include("SimulationUtils.jl")
include("OdometryUtils.jl")
include("RobotDataTypes.jl")
include("NavigationSystem.jl")

# generate canonical graphs
include("canonical/GenerateCommon.jl")
include("canonical/GenerateCircular.jl")
include("canonical/GenerateBox.jl")
include("canonical/GenerateHexagonal.jl")
include("canonical/GenerateHoneycomb.jl")
include("canonical/GenerateBeehive.jl")
include("canonical/GenerateHelix.jl")

# more utils requiring earlier functions
include("AdditionalUtils.jl")
include("g2oParser.jl")

# ScalarFields
include("services/ScalarFields.jl")

# things on their way out
include("Deprecated.jl")

# optional tools
using Requires

function __init__()
  # combining neural networks natively into the non-Gaussian  factor graph object
  @require Flux="587475ba-b771-5e3f-ad9e-33799f191a9c" begin
    @info "Loading RoME.jl tools related to Flux.jl."
    include("factors/flux/models/Pose2OdoNN_01.jl") # until a better way is found to deserialize
    include("factors/flux/MixtureFluxPose2Pose2.jl")
  end

  # Scalar field specifics

  @require ImageCore = "a09fc81d-aa75-5fe9-8630-4744c3626534" begin
    @require ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19" include("services/RequiresImages.jl")
  end
  # Images="916415d5-f1e6-5110-898d-aaa5f9f070e0"

  # @require Interpolations="a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" begin
  #   include("services/ScalarFieldsInterpolations.jl")
  # end
end

# manifold conversions required during transformation

end


############################################################
# joPDE ####################################################
############################################################

# Authors: Philipp Witte (pwitte@eos.ubc.ca), Henryk Modzelewski (hmodzelewski@eos.ubc.ca)
# Date: January 2017

export joPDE, joPDEexception, joPDEadjoint, joPDEadjointException, subsample

############################################################

# Type for linear operator representing  Pr*A(m)^-1*Ps,
# i.e. it includes source and receiver projections

# Forward
struct joPDE{DDT<:Number,RDT<:Number} <: joAbstractLinearOperator{DDT,RDT}
    name::String
    m::Integer
    n::Integer
	info::Info
	model::ModelAll
	geometry::Geometry
	options::Options
	fop::Function              # forward
	fop_T::Nullable{Function}  # transpose
end

mutable struct joPDEexception <: Exception
	msg :: String
end

# Adjoint
struct joPDEadjoint{DDT<:Number,RDT<:Number} <: joAbstractLinearOperator{DDT,RDT}
	name::String
	m::Integer
	n::Integer
	info::Info
	model::ModelAll
	geometry::Geometry
	options::Options
	fop::Function              # forward
	fop_T::Nullable{Function}  # transpose
end

mutable struct joPDEadjointException <: Exception
	msg :: String
end

############################################################
## outer constructors

function joPDE(name::String,info::Info,model::ModelAll, geometry::Geometry; options=Options(), DDT::DataType=Float32, RDT::DataType=DDT)
# JOLI wrapper for nonlinear forward modeling
	(DDT == Float32 && RDT == Float32) || throw(joPDEexception("Domain and range types not supported"))
	if typeof(geometry) == GeometryOOC
		m = sum(geometry.nsamples)
	else
		m = 0
		for j=1:info.nsrc m += length(geometry.xloc[j])*geometry.nt[j] end
	end
	n = info.n * sum(info.nt)
	if info.nsrc > 1
		srcnum = 1:info.nsrc
	else
		srcnum = 1
	end
	return F = joPDE{Float32,Float32}(name, m, n, info, model, geometry, options,
							  src -> time_modeling(model, src.geometry, src.data, geometry, [], [], srcnum, 'F', 1, options),
							  rec -> time_modeling(model, geometry, [], rec.geometry, rec.data, [], srcnum, 'F', -1, options)
							  )
end

function joPDEadjoint(name::String,info::Info,model::ModelAll, geometry::Geometry; options=Options(), DDT::DataType=Float32, RDT::DataType=DDT)
# JOLI wrapper for nonlinear forward modeling
	(DDT == Float32 && RDT == Float32) || throw(joPDEadjointException("Domain and range types not supported"))
	if typeof(geometry) == GeometryOOC
		m = sum(geometry.nsamples)
	else
		m = 0
		for j=1:info.nsrc m += length(geometry.xloc[j])*geometry.nt[j] end
	end
	n = info.n * sum(info.nt)
	if info.nsrc > 1
		srcnum = 1:info.nsrc
	else
		srcnum = 1
	end
	return F = joPDEadjoint{Float32,Float32}(name, m, n, info, model, geometry, options,
									 rec -> time_modeling(model, geometry, [], rec.geometry, rec.data, [], srcnum, 'F', -1, options),
									 src -> time_modeling(model, src.geometry, src.data, geometry, [], [], srcnum, 'F', 1, options),
								     )
end

############################################################
## overloaded Base functions

# conj(joPDE)
conj{DDT,RDT}(A::joPDE{DDT,RDT}) =
	joPDE{DDT,RDT}("conj("*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
	get(A.fop),
		A.fop_T
		)

# transpose(jo)
transpose{DDT,RDT}(A::joPDE{DDT,RDT}) =
	joPDEadjoint{DDT,RDT}(A.name,A.n,A.m,A.info,A.model,A.geometry,A.options,
		get(A.fop_T),
		A.fop
		)

# ctranspose(jo)
ctranspose{DDT,RDT}(A::joPDE{DDT,RDT}) =
	joPDEadjoint{DDT,RDT}(A.name,A.n,A.m,A.info,A.model,A.geometry,A.options,
		get(A.fop_T),
		A.fop
		)

# conj(jo)
conj{DDT,RDT}(A::joPDEadjoint{DDT,RDT}) =
    joPDEadjoint{DDT,RDT}("conj("*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
        get(A.fop),
        A.fop_T
        )

# transpose(jo)
transpose{DDT,RDT}(A::joPDEadjoint{DDT,RDT}) =
	joPDE{DDT,RDT}(A.name,A.n,A.m,A.info,A.model,A.geometry,A.options,
		get(A.fop_T),
		A.fop
		)

# ctranspose(jo)
ctranspose{DDT,RDT}(A::joPDEadjoint{DDT,RDT}) =
	joPDE{DDT,RDT}(A.name,A.n,A.m,A.info,A.model,A.geometry,A.options,
		get(A.fop_T),
		A.fop
		)

############################################################
## overloaded Base *(...jo...)

# *(joPDE,joRHS)
function *{ADDT,ARDT,vDT}(A::joPDE{ADDT,ARDT},v::joRHS{vDT})
	A.n == size(v,1) || throw(joPDEexception("shape mismatch"))
	jo_check_type_match(ADDT,vDT,join(["DDT for *(joPDE,joRHS):",A.name,typeof(A),vDT]," / "))
	V = A.fop(v)
	jo_check_type_match(ARDT,eltype(V),join(["RDT from *(joPDE,joData):",A.name,typeof(A),eltype(V)]," / "))
	return V
end

function *{ADDT,ARDT,vDT}(A::joPDEadjoint{ADDT,ARDT},v::joRHS{vDT})
	A.n == size(v,1) || throw(joPDEadjointException("shape mismatch"))
	jo_check_type_match(ADDT,vDT,join(["DDT for *(joPDE,joRHS):",A.name,typeof(A),vDT]," / "))
	V = A.fop(v)
	jo_check_type_match(ARDT,eltype(V),join(["RDT from *(joPDE,joData):",A.name,typeof(A),eltype(V)]," / "))
	return V
end

# *(joPDE,joProjection)
function *{ARDT,BDDT,CDT}(A::joPDE{CDT,ARDT},B::joProjection{BDDT,CDT})
	A.n == size(B,1) || throw(joPDEexception("shape mismatch"))
	return joModeling(A.info,A.model,B.geometry,A.geometry;options=A.options,DDT=CDT,RDT=ARDT)
end

function *{ARDT,BDDT,CDT}(A::joPDEadjoint{CDT,ARDT},B::joProjection{BDDT,CDT})
	A.n == size(B,1) || throw(joPDEadjointException("shape mismatch"))
	return joModeling(A.info,A.model,A.geometry,B.geometry,options=A.options,DDT=CDT,RDT=ARDT)'
end

# *(num,joPDE)
function *{ADDT,ARDT}(a::Number,A::joPDE{ADDT,ARDT})
	return joPDE{ADDT,ARDT}("(N*"*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
		v1 -> jo_convert(ARDT,a*A.fop(v1),false),
		v2 -> jo_convert(ADDT,a*A.fop_T(v2),false)
		)
end

function *{ADDT,ARDT}(a::Number,A::joPDEadjoint{ADDT,ARDT})
	return joPDEadjoint{ADDT,ARDT}("(N*"*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
		v1 -> jo_convert(ARDT,a*A.fop(v1),false),
		v2 -> jo_convert(ADDT,a*A.fop_T(v2),false)
		)
end

############################################################
## overloaded Basees +(...joPDE...), -(...joPDE...)

# +(joPDE,num)
function +{ADDT,ARDT}(A::joPDE{ADDT,ARDT},b::Number)
	return joPDE{ADDT,ARDT}("("*A.name*"+N)",A.m,A.n,A.info,A.model,A.geometry,A.options,
		v1 -> A.fop(v1)+joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v1,
		v2 -> A.fop_T(v2)+joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v2
		)
end

function +{ADDT,ARDT}(A::joPDEadjoint{ADDT,ARDT},b::Number)
	return joPDEadjoint{ADDT,ARDT}("("*A.name*"+N)",A.m,A.n,A.info,A.model,A.geometry,A.options,
		v1->A.fop(v1)+joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v1,
		v2->A.fop_T(v2)+joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v2
		)
end

# -(joPDE,num)
function -{ADDT,ARDT}(A::joPDE{ADDT,ARDT},b::Number)
	return joPDE{ADDT,ARDT}("("*A.name*"-N)",A.m,A.n,A.info,A.model,A.geometry,A.options,
		v1 -> A.fop(v1)-joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v1,
		v2 -> A.fop_T(v2)-joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v2
		)
end

function -{ADDT,ARDT}(A::joPDEadjoint{ADDT,ARDT},b::Number)
	return joPDEadjoint{ADDT,ARDT}("("*A.name*"-N)",A.m,A.n,A.info,A.model,A.geometry,A.options,
		v1->A.fop(v1)-joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v1,
		v2->A.fop_T(v2)-joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v2
		)
end

# -(joPDE)
-{DDT,RDT}(A::joPDE{DDT,RDT}) =
	joPDE{DDT,RDT}("(-"*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
		v1->-A.fop(v1),
		v2->-get(A.fop_T)(v2)
		)

-{DDT,RDT}(A::joPDEadjoint{DDT,RDT}) =
	joPDEadjoint{DDT,RDT}("(-"*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
		v1->-A.fop(v1),
		v2->-get(A.fop_T)(v2)
		)

############################################################
## Additional overloaded functions

# Subsample Modeling operator
function subsample{ADDT,ARDT}(F::joPDE{ADDT,ARDT}, srcnum)
	geometry = subsample(F.geometry,srcnum)		# Geometry of subsampled data container
	info = Info(F.info.n, length(srcnum), F.info.nt[srcnum])
	return joPDE(F.name, info, F.model, geometry; options=F.options, DDT=ADDT, RDT=ARDT)
end

function subsample{ADDT,ARDT}(F::joPDEadjoint{ADDT,ARDT}, srcnum)
	geometry = subsample(F.geometry,srcnum)		# Geometry of subsampled data container
	info = Info(F.info.n, length(srcnum), F.info.nt[srcnum])
	return joPDEadjoint(F.name, info, F.model, geometry; options=F.options, DDT=ADDT, RDT=ARDT)
end

getindex(F::joPDE,a) = subsample(F,a)
getindex(F::joPDEadjoint,a) = subsample(F,a)


@inline _sortperm(X::AbstractVecOrMat) = size(X,2)==1 ? sortperm(ndims(X)==1 ? X : vec(X), alg=RadixSort) : sortperm(collect(eachrow(X)))  # sort a data matrix

function Init!(o::StrBootTest{T}) where T  # for efficiency when varying r repeatedly to make CI, do stuff once that doesn't depend on r
	o.kX = o.kX₁ + o.kX₂
	o.kX₂==0 && (o.X₂ = zeros(T,o.Nobs,0))
  iszero(o.kY₂) && (o.Y₂ = zeros(T,o.Nobs,0))
  o.kZ = o.kX₁ + o.kY₂
  if o.liml && o.kX₂==o.kY₂  # exactly identified liml = 2SLS
  	o.κ = one(T)
  	o.liml = false
  end
  if !(o.REst = length(o.R₁)>0)  # base model contains no restrictions?
    o.R₁ = zeros(T,0,o.kZ)
    o.r₁ = zeros(T,0)
  end
  isnan(o.κ) && (o.κ = o.kX₂>0 ? one(T) : zero(T))  # if κ in κ-class estimation not specified, it's 0 or 1 for OLS or 2SLS
  o.WRE = !(iszero(o.κ) || o.scorebs) || o.arubin

  iszero(o.B) && (o.scorebs = true)

  o.haswt = !iszero(length(o.wt))
	if o.haswt
		o.sumwt = sum(o.wt)
		o._Nobs = o.fweights ? o.sumwt : T(o.Nobs)
	else
		o.sumwt = one(T)
		o._Nobs = T(o.Nobs)
		o.sqrtwt = T[]
	end

	o.subcluster = o.NClustVar - o.NErrClustVar

	if !(iszero(o.NClustVar) || o.issorted)
		p = _sortperm(view(o.ID, :, [collect(o.subcluster+1:o.NClustVar); collect(1:o.subcluster)]))  # sort by err cluster vars, then remaining boot cluster vars
		o.ID = ndims(o.ID)==1 ? o.ID[p] : o.ID[p,:]
		o.X₁ = ndims(o.X₁)==1 ? o.X₁[p] : o.X₁[p,:]
		o.X₂ = ndims(o.X₂)==1 ? o.X₂[p] : o.X₂[p,:]
		o.y₁ = o.y₁[p]
		o.Y₂ = ndims(o.Y₂)==1 ? o.Y₂[p] : o.Y₂[p,:]
		isdefined(o, :FEID) && nrows(o.FEID)>0 && (o.FEID = o.FEID[p])

		if o.haswt
			o.wt = o.wt[p]
			o.sqrtwt = sqrt.(o.wt)
			o.y₁ .*= o.sqrtwt  # can overwrite sorted copy of user's data
			length(o.Y₂)>0 && (o.Y₂ .*= o.sqrtwt)
			length(o.X₁)>0 && (o.X₁ .*= o.sqrtwt)
			length(o.X₂)>0 && (o.X₂ .*= o.sqrtwt)
		end
	elseif o.haswt
		o.sqrtwt = sqrt.(o.wt)
		o.y₁ = o.y₁ .* o.sqrtwt  # don't overwrite user's data
		length(o.Y₂)>0 && (o.Y₂ = o.Y₂ .* o.sqrtwt)
		length(o.X₁)>0 && (o.X₁ = o.X₁ .* o.sqrtwt)
		length(o.X₂)>0 && (o.X₂ = o.X₂ .* o.sqrtwt)
	end

  if o.WREnonARubin
	  if iszero(o.NClustVar)
	    o.info⋂ = o.info✻ = [i:i for i in 1:o.Nobs]  # no clustering, so no collapsing by cluster
	  else
	    o.info✻, o.ID✻ = panelsetupID(o.ID, 1:o.NBootClustVar)

	  end
  elseif iszero(o.NClustVar)
	  o.info✻ = [i:i for i in 1:o.Nobs]  # causes no collapsing of data in panelsum() calls
  else
	  o.info✻ = panelsetup(o.ID, 1:min(o.NClustVar,o.NBootClustVar))  # bootstrap cluster grouping defs rel to original data
  end
	o.N✻ = nrows(o.info✻)

	if o.NClustVar > o.NBootClustVar  # info for grouping by intersections of all bootstrap && clustering vars wrt data; used to speed crosstab UXAR wrt bootstrapping cluster && intersection of all error clusters
		o.info✻⋂ = panelsetup(o.ID, 1:o.NClustVar)
		if o.subcluster>0 && nrows(o.info✻) ≠ nrows(o.info✻⋂)
			throw(ErrorException("\nThis program can only perform the subcluster bootstrap when the bootstrap clusters are nested within the (intersections of the) error clusters.\n"))
		end
	else
		o.info✻⋂ = o.info✻  # info for grouping by intersections of all bootstrap && clustering vars wrt data; used to speed crosstab UXAR wrt bootstrapping cluster && intersection of all error clusters
		o.WREnonARubin && (o._ID✻⋂ = o.ID✻)
	end
	o.N✻⋂ = nrows(o.info✻⋂)

	if o.NClustVar > o.NErrClustVar  # info for intersections of error clustering wrt data
		o.info⋂ = panelsetup(o.ID, o.subcluster+1:o.NClustVar)
		ID⋂_✻⋂ = length(o.info⋂)==o.Nobs ? o.ID : o.ID[first.(o.info⋂ ),:]  # version of ID matrix with one row for each all-error-cluster-var intersection instead of 1 row for each obs; gets resorted
		o.ID✻⋂ = o.N✻⋂==o.Nobs ? o.ID : o.ID[first.(o.info✻⋂),:]  # version of ID matrix with one row for each all-bootstrap && error cluster-var intersection instead of 1 row for each obs
	else
		o.info⋂ = o.info✻⋂  # info for intersections of error clustering wrt data
		o.ID✻⋂ = ID⋂_✻⋂ = nrows(o.info⋂)==o.Nobs ? o.ID : o.ID[first.(o.info⋂),:]  # version of ID matrix with one row for each all-error-cluster-var intersection instead of 1 row for each obs; gets resorted
	end
	o.WREnonARubin && (o.info✻_✻⋂ = iszero(o.NBootClustVar) ? o.info✻⋂ : panelsetup(o.ID✻⋂, 1:o.NBootClustVar))
	o.N⋂ = nrows(o.info⋂)

	if o.bootstrapt
    if o.NClustVar>0
	    minN = T(Inf)

	    combs = [x & 2^y > 0 for x in 2^o.NErrClustVar-1:-1:1, y in o.NErrClustVar-1:-1:0]  # represent all error clustering combinations. First is intersection of all error clustering vars
	    o.clust = Vector{StrClust{T}}(undef, nrows(combs))  # leave out no-cluster combination
	    o.NErrClustCombs = length(o.clust)

	    o.BootClust = 2^(o.NClustVar - o.NBootClustVar)  # location of bootstrap clustering within list of cluster combinations

	    @inbounds for c ∈ 1:o.NErrClustCombs  # for each error clustering combination
		    ClustCols = o.subcluster .+ findall(@view combs[c,:])
		    even = isodd(length(ClustCols))  # not a typo

		    if isone(c)
		      if iszero(o.subcluster)
		    	  order = Vector{Int64}(undef,0)
		    	  info  = Vector{UnitRange{Int64}}(undef, 0)  # causes no collapsing of data in panelsum() calls
						N = o.N✻⋂
		      else
		    	  order = _sortperm(@view ID⋂_✻⋂[:,ClustCols])
		    	  info  = panelsetup(view(ID⋂_✻⋂,order,:), ClustCols)
						N = nrows(info)
		      end
		    else
		      if any(combs[c, minimum(findall(combs[c,:] .≠ combs[c-1,:])):end])  # if this sort ordering same as last to some point and missing thereafter, no need to re-sort
		    	  order = _sortperm(@view ID⋂_✻⋂[:,ClustCols])
		    	  info = panelsetup(view(ID⋂_✻⋂,order,:), ClustCols)
		      else
		    	  order = Vector{Int64}(undef,0)
						info = panelsetup(ID⋂_✻⋂, ClustCols)
		      end
					N = nrows(info)
		    end

		    minN = min(minN,N)
				multiplier = o.clusteradj && !o.clustermin ? T(N / (N-1)) : one(T)
				o.clust[c] = StrClust{T}(N, multiplier, even, order, info)
	    end

	    (o.scorebs || !o.WREnonARubin) &&
		  	(o.ClustShare = o.haswt ? @panelsum(o, o.wt, o.info⋂)/o.sumwt : T.(length.(o.info⋂)./o.Nobs)) # share of observations by group

    else  # if no clustering, cast "robust" as clustering by observation
      o.clust = [StrClust{T}(o.Nobs, o.small ? o._Nobs / (o._Nobs - one(T)) : one(T), true, Vector{Int64}(undef,0), Vector{UnitRange{Int64}}(undef,0))]
      o.NErrClustCombs = one(Int16)
      (o.scorebs || !o.WREnonARubin) &&
    		(o.ClustShare = o.haswt ? o.wt/o.sumwt : [one(T)/o.Nobs])
    end

		o.purerobust = o.robust && !o.scorebs && iszero(o.subcluster) && o.N✻==o.Nobs  # do we ever error-cluster *and* bootstrap-cluster by individual?
		o.granular   = o.WREnonARubin ? 2*o.Nobs*o.B*(2*o.N✻+1) < o.N✻*(o.N✻*o.Nobs+o.N⋂*o.B*(o.N✻+1)) :
											o.robust && !o.scorebs && (o.purerobust || (o.N⋂+o.N✻)*o.kZ*o.B + (o.N⋂-o.N✻)*o.B + o.kZ*o.B < o.N⋂*o.kZ^2 + o.Nobs*o.kZ + o.N⋂ * o.N✻ * o.kZ + o.N⋂ * o.N✻)

		if o.robust && !o.purerobust
			(o.subcluster>0 || o.granular) && !o.WREnonARubin &&
				(o.info⋂_✻⋂ = panelsetup(o.ID✻⋂, o.subcluster+1:o.NClustVar))  # info for error clusters wrt data collapsed to intersections of all bootstrapping && error clusters; used to speed crosstab UXAR wrt bootstrapping cluster && intersection of all error clusterings
			((o.scorebs && o.B>0) || (o.WREnonARubin && !o.granular && o.bootstrapt)) &&
				(o.JN⋂N✻ = zeros(T, o.N⋂, o.N✻))
		end

		if o.WREnonARubin && o.bootstrapt && !o.granular && o.NClustVar > o.NBootClustVar
			_, o._ID✻⋂ = panelsetupID(o.ID, 1:o.NClustVar)
		end
  else
	  minN = T(nrows(o.info✻))
	end

	InitFEs(o)
	if o.B>0 && o.robust && o.granular && !o.purerobust && o.bootstrapt && !o.WREnonARubin
		if o.NFE>0 && !o.FEboot
			_, o.ID✻    = panelsetupID(o.ID  , 1:o.NBootClustVar)
		else
			_, o.ID✻_✻⋂ = panelsetupID(o.ID✻⋂, 1:o.NBootClustVar)
		end
	end

	o.enumerate = o.B>0 && o.auxtwtype == :rademacher && o.N✻*log(2) < log(o.B)+1e-6  # generate full Rademacher set?
	o.enumerate && (o.maxmatsize = 0)

	o.Nw = iszero(o.maxmatsize) ? one(Int64) : ceil(Int64, (o.B+1) * Float64(max(nrows(o.ID✻), length(o.ID✻_✻⋂), o.N✻) * sizeof(T)) / o.maxmatsize / 1073741824) # 1073741824 = giga(byte)
	if isone(o.Nw)
		MakeWildWeights!(o, o.B, first=true)  # make all wild weights, once
		o.enumerate && (o.B = ncols(o.v) - 1)  # replications reduced to 2^G
		o.WeightGrp = [1:ncols(o.v)]
	else
    o.seed = rand(o.rng, UInt64)
		_B = ceil(Int64, (o.B+1) / o.Nw)
		o.Nw = ceil(Int64, (o.B+1) / _B)
		o.WeightGrp = [(i-1)*_B+1:i*_B for i ∈ 1:o.Nw]
		o.WeightGrp[end] = first(o.WeightGrp[end]):o.B+1
	end

	if o.ml
		o.dof = nrows(o.R)
	else
		if o.arubin
			o.R = hcat(zeros(o.kX₂,o.kX₁), Matrix(I(o.kX₂)))  # attack surface is all endog vars
			o.R₁ = o.kX₁>0 && nrows(o.R₁)>0 ? hcat(o.R₁[:,1:o.kX₁], zeros(nrows(o.R₁),o.kX₂)) : zeros(0, o.kX)  # and convert model constraints from referring to X₁, Y₂ to X₁, X₂
		end
		o.dof = nrows(o.R)

		if !o.WRE && iszero(o.κ)  # regular OLS
			o.DGP = StrEstimator{T}(true, o.liml, o.fuller, o.κ)
			o.Repl = StrEstimator{T}(true, false, zero(T), zero(T))  # XXX isDGP=1 for Repl? doesn't matter?
			setR!(o.DGP, o, o.null ? [o.R₁ ; o.R] : o.R₁)  # DGP constraints: model constraints + null if imposed
			setR!(o.Repl, o, o.R₁)  # model constraints only
			InitVarsOLS!(o.DGP, o, o.Repl.R₁perp)
			InitTestDenoms!(o.DGP, o)
			o.M = o.DGP  # StrEstimator object from which to get A, AR, XAR
		elseif o.arubin
			if o.willplot  # for plotting/CI purposes get original point estimate since not normally generated
				o.DGP = StrEstimator{T}(true, o.liml, o.fuller, o.κ)
				setR!(o.DGP, o, o.R₁, zeros(T,0,o.kZ))  # no-null model
				InitVarsIV!(o.DGP, o)
				EstimateIV!(o.DGP, o, o.r₁)
				o.confpeak = o.DGP.β̈  # estimated coordinate of confidence peak
			end

			o.DGP = StrEstimator{T}(true, false, zero(T), zero(T))
			setR!(o.DGP, o, o.R₁)
			InitVarsARubin!(o.DGP, o)
			InitTestDenoms!(o.DGP, o)
			o.M = o.DGP  # StrEstimator object from which to get A, AR, XAR
			o.kZ = o.kX

		elseif o.WREnonARubin
			o.Repl = StrEstimator{T}(false, o.liml, o.fuller, o.κ)
			setR!(o.Repl, o, o.R₁, o.R)
			InitVarsIV!(o.Repl, o)
			EstimateIV!(o.Repl, o, o.r₁)

			o.DGP = StrEstimator{T}(true, T(o.kX₂ ≠ o.kY₂), zero(T), one(T))
			setR!(o.DGP, o, o.null ? [o.R₁ ; o.R] : o.R₁, zeros(T,0,o.kZ))  # DGP constraints: model constraints + null if imposed
			InitVarsIV!(o.DGP, o)
			if !o.null  # if not imposing null, then DGP constraints, κ, Hessian, etc. do not vary with r and can be set now
				EstimateIV!(o.DGP, o, o.r₁)
				MakeResidualsIV!(o.DGP, o)
			end

			InitWRE!(o)

		else  # the score bootstrap for IV/GMM uses a IV/GMM DGP but then masquerades as an OLS test because most factors are fixed during the bootstrap. To conform, need DGP and Repl objects with different R, R₁, one with FWL, one not

			o.DGP = StrEstimator{T}(true, o.liml, o.fuller, o.κ)
			setR!(o.DGP, o, o.null ? [o.R₁ ; o.R] : o.R₁, zeros(T,0,o.kZ))  # DGP constraints: model constraints + null if imposed
			InitVarsIV!(o.DGP, o)
			o.Repl = StrEstimator{T}(true, o.liml, o.fuller, o.κ)
			setR!(o.Repl, o, o.R₁, I)  # process replication restraints = model constraints only
			InitVarsIV!(o.Repl, o, o.Repl.R₁perp)
			EstimateIV!(o.Repl, o, o.r₁)  # bit inefficient to estimate in both objects, but maintains the conformity
			InitTestDenoms!(o.Repl, o)
			o.M = o.Repl  # StrEstimator object from which to get A, AR, XAR; DGP follows WRE convention of using FWL, Repl follows OLS convention of not; scorebs for IV/GMM mixes the two
			if !o.null  # if not imposing null, then DGP constraints, κ, Hessian, etc. do not vary with r and can be set now
				EstimateIV!(o.DGP, o, o.r₁)
				MakeResidualsIV!(o.DGP, o)
			end
		end
  end

  if !o.WREnonARubin && o.bootstrapt
		o.denom = [Matrix{T}(undef,0,0) for _ in 1:o.dof, _ in 1:o.dof]
		if o.robust
			o.Kcd =                       Matrix{Matrix{T}}(undef, o.NErrClustCombs, o.dof)
			o.Jcd = iszero(o.B) ? o.Kcd : Matrix{Matrix{T}}(undef, o.NErrClustCombs, o.dof)  # if B = 0, Kcd will be multiplied by v, which is all 1's, and will constitute Jcd
		end

		if o.robust && o.granular<o.NErrClustCombs && o.B>0
			inds = o.subcluster>0 ?
				        [CartesianIndex(j, i) for (j,v) ∈ enumerate(o.info⋂_✻⋂) for i ∈ v] :  # crosstab ∩,* is wide
								o.NClustVar == o.NBootClustVar ?
										[CartesianIndex(i, i) for i ∈ 1:o.N✻⋂] :  # crosstab ∩,* is square
										[CartesianIndex(i, j) for (j,v) ∈ enumerate(o.clust[o.BootClust].info) for i ∈ v]  # crosstab ∩,* is tall
			o.crosstab⋂✻ind = LinearIndices(FakeArray(Tuple(max(inds...))...))[inds]
		end
	end

  o.sqrt = isone(o.dof)  # work with t/z stats instead of F/chi2?

  if o.small
		_dof = o._Nobs - o.kZ + (o.ml ? 0 : ncols(o.Repl.R₁invR₁R₁)) - o.FEdfadj
		o.dof_r = o.NClustVar>0 ? minN - one(T) : _dof  # floating-point dof_r allows for fractional fweights, FWIW...
		o.smallsample = _dof / (o._Nobs - o.robust)
	else
		o.smallsample = one(T)
		o.dof_r = zero(T)
	end
	o.clustermin && (o.smallsample *= (minN - 1) / minN)  # ivreg2-style adjustment when multiway clustering
	o.multiplier = o.small ? o.smallsample / o.dof : o.smallsample  # divide by # of constraints because F stat is so defined
  !(o.robust || o.ml) && (o.multiplier *= o._Nobs)  # will turn sum of squared errors in denom of t/z into mean
  o.sqrt && (o.multiplier = √o.multiplier)

	o.dist = fill(T(NaN), 1, o.B+1)
  (o.Nw>1 || o.WREnonARubin || (!o.null && o.dof≤2)) && (o.numer = fill(T(NaN), o.dof, o.B+1))

  if !o.WREnonARubin
		o.poles = o.anchor = zeros(T,0)
		o.interpolable = o.bootstrapt && o.null && o.Nw==1 && (iszero(o.κ) || o.arubin)
		o.interpolate_u = !(o.robust || o.ml)
		if o.interpolable
			o.∂numer∂r = Vector{Matrix{T}}(undef, o.q)
			o.interpolate_u && (o.∂u∂r = Vector{Matrix{T}}(undef, o.q))
			if o.robust
				o.∂denom∂r   = fill(Matrix{T}(undef,0,0), o.q, o.dof, o.dof)
				o.∂²denom∂r² = fill(Matrix{T}(undef,0,0), o.q, o.q, o.dof, o.dof)
				o.∂Jcd∂r     = fill(Matrix{T}(undef,0,0), o.q, o.NErrClustCombs, o.dof)
			end
		end
  end
	o.initialized = true
	nothing
end

function InitFEs(o::StrBootTest{T}) where T
	if isdefined(o, :FEID) && length(o.FEID)>0
		p = _sortperm(o.FEID)
		sortID = o.FEID[p]
		i_FE = 1; o.FEboot = o.B>0 && !o.WREnonARubin && o.NClustVar>0; j = o.Nobs; o._FEID = ones(Int64, o.Nobs)
		o.invFEwt = zeros(T, o.NFE>0 ? o.NFE : o.Nobs)
		o.FEs = Vector{StrFE{T}}(undef, o.NFE>0 ? o.NFE : o.Nobs)
		_sqrtwt = T[]
		@inbounds for i ∈ o.Nobs-1:-1:1
			if sortID[i] ≠ sortID[i+1]
				is = @view p[i+1:j]
				if o.haswt
					_sqrtwt  = @view o.sqrtwt[is]
					wt = _sqrtwt / (sumFEwt = sum(@view o.wt[is]))
				else
					sumFEwt = T(j - i)
					wt = [one(T)/sumFEwt]
				end
				o.FEs[i_FE] = StrFE{T}(is, wt, _sqrtwt)
				((o.B>0 && o.robust && o.granular < o.NErrClustVar) || (o.WREnonARubin && o.robust && o.granular && o.bootstrapt)) &&
					(o.invFEwt[i_FE] = one(T) / sumFEwt)

				j = i

				if o.FEboot  # are all of this FE's obs in same bootstrapping cluster? (But no need to check if B=0 for then CT(W.*E) in 2nd term of (62) orthogonal to v = col of 1's)
					tmp = o.ID[is, 1:o.NBootClustVar]
					o.FEboot = all(tmp .== view(tmp, 1,:)')
				end
				i_FE += 1
			end
			o._FEID[p[i]] = i_FE
		end
		is = @view p[1:j]
		if o.haswt
			_sqrtwt  = @view o.sqrtwt[is]
			wt = _sqrtwt / (sumFEwt = sum(@view o.wt[is]))
		else
			sumFEwt = T(j)
			wt = T[1/sumFEwt]
		end
		o.FEs[i_FE] = StrFE{T}(is, wt, _sqrtwt)
		o.robust && ((o.B>0 && o.granular < o.NErrClustVar) || (o.WREnonARubin && o.granular && o.bootstrapt)) &&
			(o.invFEwt[i_FE] = 1 / sumFEwt)

		if iszero(o.NFE)
			o.NFE = i_FE
			resize!(o.invFEwt, o.NFE)
			resize!(o.FEs    , o.NFE)
		end
		o.FEdfadj==-1 && (o.FEdfadj = o.NFE)

		if o.FEboot  # are all of this FE's obs in same bootstrapping cluster?
			tmp = o.ID[is, 1:o.NBootClustVar]
			o.FEboot = all(tmp .== @view tmp[1,:])
		end

		if o.robust && o.B>0 && o.bootstrapt && !o.FEboot && o.granular < o.NErrClustVar
			o.infoBootAll = panelsetup(o.ID✻⋂, 1:o.NBootClustVar)  # info for bootstrapping clusters wrt data collapsed to intersections of all bootstrapping && error clusters
		end

		o.X₁ = partialFE(o, o.X₁)  # don't overwrite caller's data
		o.X₂ = partialFE(o, o.X₂)
		o.y₁ = partialFE(o, o.y₁)
		o.Y₂ = partialFE(o, o.Y₂)
	else
		o.FEdfadj = 0
	end
	nothing
end

# draw wild weight matrix of width _B. If first=true, insert column of 1s at front. For non-Anderson-Rubin WRE, subtract 1 from all
const ϕ = (1 + √5)/2

function MakeWildWeights!(o::StrBootTest{T}, _B::Integer; first::Bool=true) where T
  if _B>0  # in scoretest or waldtest WRE, still make v a col of 1's
    o.v = o.enumerate ? o.WREnonARubin ? [zeros(o.N✻) count_binary(o.N✻, -2, 0)] :  # complete Rademacher set
								                         [ones( o.N✻) count_binary(o.N✻, -1, 1)] :
					o.auxtwtype == :normal ? randn(o.rng, T, o.N✻, _B+first) .- o.WREnonARubin :
					o.auxtwtype == :gamma  ? quantile.(Gamma{T}(4,.5),  rand(o.rng, T, o.N✻, _B+first)) .- (2 - o.WREnonARubin) :
					o.auxtwtype == :webb   ? rand(o.rng, T.([-√1.5, -1, -√.5, √.5, 1, √1.5] .- o.WREnonARubin), o.N✻, _B+first) :
				  o.auxtwtype == :mammen ? getindex.(Ref(T.([1-ϕ; ϕ] .- o.WREnonARubin)), ceil.(Int16, rand(o.rng, o.N✻, _B+first) ./ (ϕ/√5))) :
		      o.WREnonARubin         ? -2rand(o.rng, Bool, o.N✻, _B+first) : # Rademacher
			                             (o.v_sd = .5; rand(o.rng, Bool, o.N✻, _B+first) .- T(.5))
		first && !(o.enumerate && isone(o.v_sd)) && (o.v[:,1] .= o.WREnonARubin ? zero(T) : o.v_sd)  # keep original residuals in first entry to compute base model stat
  else
		o.v = Matrix{T}(undef,0,1)  # in places, ncols(v) indicates B -- 1 for classical tests
  end
	nothing
end


function get_files(dir::String)
    return collect(Base.Iterators.flatten([files for (_, _, files) in walkdir(dir)]))
end

function get_root_and_files(dir::AbstractString)
    allfiles = Matrix{Int}(undef, 0, 2)

    allfilesraw = [(root, files) for (root, _, files) in walkdir(dir)]

    for (objectdir, objects) in allfilesraw
        for object in objects
            allfiles = cat(allfiles, [objectdir object], dims = 1)
        end
    end

    return allfiles
end

function check_uniqueness(list::AbstractVector)
    processed = []
    for i in list
        i_parsed = split(i, '_')[1]
        if i_parsed ∈ processed
            println("We have a $(i_parsed) duplicate")
            continue
        end
        push!(processed, i_parsed)
    end
end

# check_uniqueness(get_files("/Users/jakeireland/Desktop/New original/"))
check_uniqueness(get_files("/Users/jakeireland/projects/GorillaSCExperiment/data/Original"))

function main()
    # selected_images = []
    # selected_images = get_files("/Users/jakeireland/projects/GorillaSCExperiment/data/chosen_object_images_1/")
    selected_images = vcat(get_files("/Users/jakeireland/projects/GorillaSCExperiment/data/chosen_object_images_1/"), get_files("/Users/jakeireland/projects/GorillaSCExperiment/data/chosen_object_images_2/"), get_files("/Users/jakeireland/projects/GorillaSCExperiment/data/chosen_object_images_3/"), get_files("/Users/jakeireland/projects/GorillaSCExperiment/data/chosen_object_images_4/"), get_files("/Users/jakeireland/projects/GorillaSCExperiment/data/chosen_object_images_5/"), get_files("/Users/jakeireland/projects/GorillaSCExperiment/data/chosen_object_images_6/"))
    allfiles = get_root_and_files("/Users/jakeireland/projects/GorillaSCExperiment/data/THINGS/")

    i = 0
    while i < 600
        randimageindex = rand(1:size(allfiles, 1))
        randimageinfo = allfiles[randimageindex, :]
        randimage = randimageinfo[2];
        # println(randimageinfo)
        if match(r"\.jpg", randimage).match != ""
            if randimage ∉ selected_images
                # println("passed")
                i += 1
                push!(selected_images, randimage)
                cp(joinpath(randimageinfo[1], randimage), joinpath("/Users/jakeireland/projects/GorillaSCExperiment/data/chosen_object_images_7", randimage))
            end
        end
    end

    return nothing
end

# main()


### Make lexicon file from Corpus to then make MinimalistGrammar
#

##################CREATE POSSIBLE FEATURE SETS GIVE 1 CAT###########################
function add_feats(feature_list, sub_list,category_feats, k)
    append!(feature_list, sub_list)
    if k == 0
        return feature_list
    end
    new_sub_list = []
    for list in sub_list
        for c in category_feats
            Rfeats = string(list[1:(end-1)], "R=", c, " ", list[end])
            Lfeats = string("L=", c, " ", list)
            push!(new_sub_list, Rfeats)
            push!(new_sub_list, Lfeats)
        end
    end
    k-=1
    return add_feats(feature_list,new_sub_list,category_feats,k)
end

function add_feats_Rightfirst(feature_list, sub_list,category_feats, k)
    append!(feature_list, sub_list)
    if k == 0
        return feature_list
    end
    new_sub_list = []
    for list in sub_list
        for c in category_feats
            Rfeats = string("R=", c, " ", list)
            push!(new_sub_list, Rfeats)
        end
    end
    k-=1
    return add_feats_Rightfirst(feature_list,new_sub_list,category_feats,k)
end

function add_feats_Leftsecond(feature_list, sub_list,category_feats, k, m)
    append!(feature_list, sub_list)
    if k == 0
        return feature_list
    end
    new_sub_list = []
    for list in sub_list
        if length(list) < 4*m+1
            for c in category_feats
                Lfeats = string("L=", c, " ", list)
                push!(new_sub_list, Lfeats)
            end
        end
    end
    k-=1
    return add_feats_Leftsecond(feature_list,new_sub_list,category_feats,k, m)
end

function add_feats_Balanced(feature_list, sub_list,category_feats, k, m)
    right_list = add_feats_Rightfirst(feature_list, sub_list, category_feats, k)
    left_list = add_feats_Leftsecond([], right_list, category_feats, k, m)
    return left_list
end

###################CREATE FEATURE SET DICT FOR ALL CATEGORIES###################
function make_feature_list_by_cat_dict(categories)
    feature_list = Dict()
    cats = []
    for (c, sels, k) in categories
        push!(cats, c)
        sub_list = [c]
        feature_list[c] = add_feats_Balanced([], sub_list , sels , k, k)
    end
    #Add start symbol category
    feature_list["c"] = add_feats_Rightfirst([], ["c"] , cats, 1)
    return feature_list
    #feature_list_start = add_feats_Balanced([], ["c"] , cats, 2, 2)
    #return (feature_list, feature_list_start)
end

function make_feature_list_by_cat(categories, k)
    feature_list = Dict()
    for c in categories
        sub_list = [c]
        feature_list[c] = add_feats([], sub_list , categories, k)
    end
    # Add start symbol category
    feature_list_start = add_feats([], ["c"] , categories, k)
    return (feature_list, feature_list_start)
end

function make_feature_list(categories, k)
    feature_list = add_feats([], categories , categories, k)
    # Add start symbol category
    feature_list_start = add_feats([], ["c"] , categories, k)
    return (feature_list, feature_list_start)
end
######################TRANSFORM POS TO MINIMALIST CAT###########################

function getcat_fromPOS(pos)
    cat = ""
    if in(pos, ["NOUN", "PROPN"])
        cat = "n"
    elseif in(pos, ["DET", "PRON"])
        cat = "d"
    elseif in(pos, ["VERB"])
        cat = "v"
    elseif in(pos, ["ADJ"])
        cat = "a"
    elseif in(pos, ["ADV"])
        cat = "x"
    elseif in(pos, ["ADP"])
        cat = "p"
    elseif in(pos, ["AUX"])
        cat = "t"
    elseif in(pos, ["SCONJ"])
        cat = "s"
    elseif in(pos, ["PART"])
        cat = "r"
    end
    return cat
end

## In the case of English, this splits the POS PART into three category types:
## infinitive-> t, possessive-> d, negation-> neg
function getcat_fromPOS_EnglishOnly(lemma, pos)
    cat = ""
    if in(pos, ["NOUN", "PROPN"])
        cat = "n"
    elseif in(pos, ["DET", "PRON"])
        cat = "d"
    elseif in(pos, ["VERB"])
        cat = "v"
    elseif in(pos, ["ADJ"])
        cat = "a"
    elseif in(pos, ["ADV"])
        cat = "x"
    elseif in(pos, ["ADP"])
        cat = "p"
    elseif in(pos, ["AUX"])
        cat = "t"
    elseif in(pos, ["SCONJ"])
        cat = "s"
    elseif in(pos, ["PART"])
        if contains(lemma, "n")
            cat = "neg"
        elseif contains(lemma, "t")
            cat = "t"
        else
            cat = "d"
        end
    end
    return cat
end

######################CREATE A LEXICON FROM CORPUS##############################

## for testing, corpus is a list of sentences, which is itself a list of tuples,
## (word, cat)
function makeLexicon_test(corpus, categories, k)

    feature_list, feature_list_start = make_feature_list_by_cat(categories, k)
    words = []
    lexicon = ""
    for featlist in feature_list_start
        item = string( ":: ", featlist, " ; 1")
        lexicon = string(lexicon,"\n", item)
    end

    for sent in corpus
        for (word,cat) in sent
            if !(word in words)
                push!(words, word)
                for featlist in feature_list[cat]
                    item = string(word, " :: ", featlist, " ; 1")
                    lexicon = string(lexicon,"\n", item)
                end
            end
        end
    end
    open("lexicon.txt", "w") do f
        write(f, lexicon)
    end
    return lexicon
end

## Makes a lexicon and filtered corpus from UD data. If using the english only
## category mapping add eng_only=true to call.
function makeLexicon_andCorpus(UDcorpus, categories, eng_only=false)

    feature_list = make_feature_list_by_cat_dict(categories)
    words = []
    lexicon = ""
    corpus = []
    counts = Dict()
    for (cat, sels, n) in categories
        counts[cat] = 1
    end
    for (cat,featlist) in feature_list
        for flist in featlist
            item = string( ":: ", flist, " ; 1")
            lexicon = string(lexicon,"\n", item)
        end
    end
    for sent in UDcorpus.sents
        sentence = []
        goodsent = true
        for word in sent.words
            w = lowercase(word.form)
            ##remove sentences which contain one of these POS tags
            if in(word.upos_tag, ["INTJ", "CONJ","SYM","X"])
                goodsent = false
                break
            elseif ';' in word.form
                goodsent = false
                break
            elseif in(word.upos_tag, ["ADJ","ADV","ADP","AUX","DET", "NOUN", "PART", "PRON", "PROPN", "SCONJ", "VERB"])
                if !(string(w,word.upos_tag) in words)
                    push!(words, string(w,word.upos_tag))
                    if eng_only
                        cat = getcat_fromPOS_EnglishOnly(word.lemma, word.upos_tag)
                    else
                        cat = getcat_fromPOS(word.upos_tag)
                    end
                    counts[cat] +=1
                    for featlist in feature_list[cat]
                        item = string(w, " :: ", featlist, " ; 1")
                        lexicon = string(lexicon,"\n", item)
                    end
                    sentence = push!(sentence, w)
                else
                    sentence = push!(sentence, w)
                end

            ### ADDING SECTION TO INCLUDE NUMERALS
            elseif in(word.upos_tag, ["NUM"])
                if !(string(w,word.upos_tag) in words)
                    push!(words, string(w,word.upos_tag))
                    cats = ["d","n","a"]
                    for cat in cats
                        counts[cat] +=1
                        for featlist in feature_list[cat]
                            item = string(w, " :: ", featlist, " ; 1")
                            lexicon = string(lexicon,"\n", item)
                        end
                    end
                    sentence = push!(sentence, w)
                else
                    sentence = push!(sentence, w)
                end
            end
        end
        if goodsent && sentence != []
            push!(corpus, sentence)
        end
    end
    open("lexicon.txt", "w") do f
        write(f, lexicon)
    end
    open("corpus.txt", "w") do f
        for sentence in corpus
            for word in sentence
                write(f, string(word, " "))
            end
            write(f, "\n")
        end
    end
    corpus = convert(Array{Array{String,1},1},corpus)
    return (lexicon, corpus, counts)
end

########## FUNCTIONS FOR FIXING GRAMMARS ##########

# remove duplicate lexical items
function remove_duplicates!(grammar)
    for word in keys(grammar.lexicon)
        ids = unique(map(GeneralizedChartparsing.id, grammar.lexicon[word]))
        if length(ids) != length(grammar.lexicon[word])
            grammar.lexicon[word] = map(LexicalItem, ids)
        end
    end
end

# remove null heads of the form =x x (vacuous heads)
function remove_vacuous_nullheads!(grammar)
    function nonvacuous(li)
        feats = li.features
        if length(feats) == 2
            GeneralizedChartparsing.name(feats[1]) != GeneralizedChartparsing.name(feats[2])
        else true
        end
    end
    grammar.lexicon[""] = filter(nonvacuous, grammar.lexicon[""])
end


# A utility script for running a comparison benchmark.

using PkgBenchmark

import BenchmarkTools:
    BenchmarkGroup,
    TrialJudgement,
    prettydiff,
    memory,
    ratio,
    isregression

function flatten(group::BenchmarkGroup; prefix="")
    results = Pair[]

    for (name, val) in group.data
        if isa(val, BenchmarkGroup)
            append!(results, flatten(val; prefix="$prefix/$name"))
        elseif isa(val, TrialJudgement)
            push!(results, "$prefix/$name" => val)
        end
    end

    return results
end

@testset "Benchmarks" begin
    curr_commit = get(ENV, "MEMENTO_CURR_COMMIT", "16b620c")
    base_commit = get(ENV, "MEMENTO_BASE_COMMIT", "be07e98")
    result = judge(
        "Memento",
        curr_commit,
        base_commit;
        saveresults=false,
        promptsave=false,
        f=median,
        judgekwargs=Dict(
            :time_tolerance => 0.2,
            :memory_tolerance => 0.2,
        ),
    )

    flat_result = flatten(result["Memento"])

    println("Benchmark, Time, Memory")
    for element in flat_result
        println(string(
            element.first,
            ", ",
            prettydiff(time(ratio(element.second))),
            " ( ",
            time(element.second),
            " )",
            ", ",
            prettydiff(memory(ratio(element.second))),
            " ( ",
            memory(element.second),
            " )",
        ))
    end

    for element in flat_result
        @test !isregression(element.second)
    end
end


function initialize_pss!(
    device_states,
    static::PSY.StaticInjection,
    dynamic_device::PSY.DynamicGenerator{M, S, A, TG, PSY.PSSFixed},
) where {M <: PSY.Machine, S <: PSY.Shaft, A <: PSY.AVR, TG <: PSY.TurbineGov} end

#Currently not working properly.
#=
function initialize_pss!(
    device_states,
    static::PSY.StaticInjection,
    dynamic_device::PSY.DynamicGenerator{M, S, A, TG, PSY.PSSSimple},
) where {M <: PSY.Machine, S <: PSY.Shaft, A <: PSY.AVR, TG <: PSY.TurbineGov}

    ω0 = PSY.get_ext(dynamic_device)[CONTROL_REFS][ω_ref_index]
    P0 = PSY.get_active_power(static)
    τe = get_inner_vars(dynamic_device)[τe_var]

    #Get parameters
    pss = PSY.get_pss(dynamic_device)
    K_ω = PSY.get_K_ω(pss)
    K_p = PSY.get_K_p(pss)

    #It assumes that the system is balanced so:
    get_inner_vars(dynamic_device)[V_pss_var] = K_ω * (ω0 - ω0) + K_p * (ω0 * τe - P0)

end
=#


module IncrementalInference

@info "Multithreaded  convolutions possible, Threads.nthreads()=$(Threads.nthreads()).  See `addFactor!(.;threadmodel=MultiThreaded)`."

using Distributed
using Requires
using Reexport

@reexport using Distributions
@reexport using KernelDensityEstimate
@reexport using ApproxManifoldProducts
@reexport using Graphs
@reexport using LinearAlgebra

using
  Dates,
  DistributedFactorGraphs,
  Statistics,
  Random,
  NLsolve,
  StatsBase,
  JLD2,
  FileIO,
  ProgressMeter,
  DocStringExtensions,
  FunctionalStateMachine,
  Optim, # might be deprecated in favor for only NLsolve dependency
  JSON2

using Logging

const KDE = KernelDensityEstimate
const AMP = ApproxManifoldProducts
const DFG = DistributedFactorGraphs
const FSM = FunctionalStateMachine

import Base: convert
# import HDF5: root
import Distributions: sample
import Random: rand, rand!
import KernelDensityEstimate: getBW
import ApproxManifoldProducts: kde!, manikde!
import DistributedFactorGraphs: addVariable!, addFactor!, ls, lsf, isInitialized, hasOrphans, compare, compareAllSpecial

# missing exports
import DistributedFactorGraphs: PackedFunctionNodeData, FunctionNodeData

# TODO temporary for initial version of on-manifold products
KDE.setForceEvalDirect!(true)

export
  KDE,
  AMP,
  DFG,
  FSM,

  # DFG SpecialDefinitions
  AbstractDFG,
  hasVariable,
  getSolverParams,

  *,
  notifyCSMCondition,
  CSMHistory,
  getTreeCliqsSolverHistories,
  assignTreeHistory!,

  updateCliqSolvableDims!,
  fetchCliqSolvableDims,
  BayesTreeNodeData,

  # state machine methods
  StateMachine,
  exitStateMachine,
  getCliqSolveHistory,
  filterHistAllToArray,
  cliqHistFilterTransitions,
  printCliqSummary,
  printCliqHistorySummary,
  printGraphSummary,
  printSummary,
  getGraphFromHistory,
  getCliqSubgraphFromHistory,
  sandboxStateMachineStep,
  sandboxCliqResolveStep,
  # draw and animate state machine
  getStateLabel,
  histStateMachineTransitions,
  histGraphStateMachineTransitions,
  drawStateTransitionStep,
  drawStateMachineHistory,
  animateStateMachineHistoryByTime,
  animateStateMachineHistoryByTimeCompound,
  animateCliqStateMachines,
  csmAnimate,
  makeCsmMovie,
  lockUpStatus!,
  unlockUpStatus!,
  lockDwnStatus!,
  unlockDwnStatus!,
  getSiblingsDelayOrder,
  areSiblingsRemaingNeedDownOnly,

  # general types for softtyping of variable nodes
  InferenceVariable,
  ContinuousScalar,
  ContinuousMultivariate,
  SamplableBelief,
  Prior,
  PackedPrior,
  MsgPrior,
  PackedMsgPrior,
  PartialPrior,
  PackedPartialPrior,
  LinearConditional,
  PackedLinearConditional,
  MixturePrior,
  PackedMixturePrior,
  MixtureLinearConditional,
  PackedMixtureLinearConditional,

  ls2,
  lsRear,
  # from DFG
  ls,
  lsf,
  getVariableIds,
  calcVariablePPE,
  sortVarNested,
  hasOrphans,
  drawCopyFG,
  isVariable,
  isFactor,
  # from dfg
  getfnctype,
  getFactorType,
  getSofttype,
  getVariableType,
  lsfPriors,
  isPrior,
  lsTypes,
  lsfTypes,

  # using either dictionary or cloudgraphs
  # VariableNodeData,
  # PackedVariableNodeData,
  FactorMetadata,
  getpackedtype,
  encodePackedType,
  FunctionNodeData,
  PackedFunctionNodeData, # moved to DFG
  encodePackedType,
  decodePackedType,
  normalfromstring,
  categoricalfromstring,
  extractdistribution,

  FactorGraph,
  SolverParams,
  addNode!,
  addVariable!,
  deleteVariable!,
  addFactor!,
  deleteFactor!,
  addMsgFactors!,
  deleteMsgFactors!,
  factorCanInitFromOtherVars,
  doautoinit!,
  manualinit!,
  initVariable!,
  asyncTreeInferUp!,
  initInferTreeUp!,
  solveCliqWithStateMachine!,
  resetData!,
  resetTreeCliquesForUpSolve!,
  resetFactorGraphNewTree!,
  setVariableInitialized!,
  setVariableInferDim!,
  resetVariable!,
  resetCliqSolve!,
  getFactor,
  getFactorDim,
  getVariableDim,
  getVariableInferredDim,
  getVariableInferredDimFraction,
  getVariablePotentialDims,
  getVariableSolvableDim,
  getFactorSolvableDim,
  getFactorInferFraction,
  getCliqSiblingsPriorityInitOrder,
  isCliqFullDim,
  hasFactor,
  getVariable,
  # getVert, # deprecated use DFG.getVariable getFactor instead
  getData,
  setData!,
  getManifolds,
  getVarNode,
  getVal,
  getBW,
  setVal!,
  getNumPts,
  getBWVal,
  setBW!,
  setValKDE!,
  buildCliqSubgraphUp,
  buildCliqSubgraphDown,
  setCliqUpInitMsgs!,
  cliqInitSolveUpByStateMachine!,

  # state machine functions
  finishCliqSolveCheck_StateMachine,
  doCliqInferAttempt_StateMachine,
  determineCliqNeedDownMsg_StateMachine,
  blockUntilChildrenStatus_StateMachine,
  blockUntilSiblingsStatus_StateMachine,
  doesCliqNeeddownmsg_StateMachine,
  slowCliqIfChildrenNotUpsolved_StateMachine,
  whileCliqNotSolved_StateMachine,
  buildCliqSubgraph_StateMachine,
  isCliqUpSolved_StateMachine,
  determineAllChildrenNeedDownMsg_StateMachine,
  testCliqCanRecycled_StateMachine,
  buildCliqSubgraphForDown_StateMachine,

  #
  isPartial,
  isInitialized,
  isTreeSolved,
  isUpInferenceComplete,
  isCliqInitialized,
  isCliqReadyInferenceUp,
  isCliqUpSolved,
  areCliqVariablesAllInitialized,
  areCliqChildrenNeedDownMsg,
  areCliqChildrenAllUpSolved,
  ensureAllInitialized!,
  doCliqAutoInitUpPart1!,
  doCliqAutoInitUpPart2!,
  doCliqInitDown!,
  cycleInitByVarOrder!,
  doCliqUpSolve!,
  getCliqInitUpMsgs,
  getCliqStatus,
  setCliqStatus!,
  getSolveCondition,
  prepCliqInitMsgsUp,
  prepCliqInitMsgsDown!,
  updateFullVert!,
  getOutNeighbors,
  BayesTree,
  EasyMessage,
  NBPMessage,
  FullExploreTreeType,
  ExploreTreeType,
  FactorGraph,
  initfg,
  buildSubgraphFromLabels,
  subgraphShortestPath,
  transferUpdateSubGraph!,
  getEliminationOrder,
  buildBayesNet!,
  emptyBayesTree,
  buildTree!,
  buildTreeFromOrdering!,
  resetBuildTreeFromOrder!,
  prepBatchTree!,
  wipeBuildNewTree!,
  buildCliquePotentials,
  hasCliq,
  getCliq,
  whichCliq,
  getCliqDepth,
  getTreeAllFrontalSyms,
  getTreeCliqUpMsgsAll,
  stackCliqUpMsgsByVariable,
  getCliqChildMsgsUp,
  getCliqParentMsgDown,
  getCliqDownMsgsAfterDownSolve,
  isReadyCliqInferenceUp,
  childCliqs,
  getChildren,
  parentCliq,
  getParent,
  getCliqSiblings,
  getKDE,
  getVertKDE,
  initializeNode!,
  CliqStateMachineContainer,
  batchSolve!,
  solveTree!,
  solveCliq!,
  fifoFreeze!,
  getCurrentWorkspaceFactors,
  getCurrentWorkspaceVariables,

  # temp const types TODO
  TempBeliefMsg,
  TempUpMsgPlotting,

  #functors need
  getSample,
  freshSamples!,

  #Visualization
  writeGraphPdf, # deprecated, but first move code to drawGraph before deleting
  drawGraph,
  drawGraphCliq,
  drawCliqSubgraphUpMocking,
  drawTree,
  printgraphmax,
  # allnums,
  # isnestednum,
  # sortnestedperm,

  # Bayes (Junction) Tree
  evalPotential,
  evalFactor2,
  approxConv,
  approxConvBinary,

  # more debugging tools
  localProduct,
  treeProductUp,
  approxCliqMarginalUp!,
  dontMarginalizeVariablesAll!,
  unfreezeVariablesAll!,
  resetVariableAllInitializations!,
  isMarginalized,
  isMultihypo,
  getMultihypoDistribution,
  getHypothesesVectors,
  isCliqMarginalizedFromVars,
  isCliqParentNeedDownMsg,
  setCliqAsMarginalized!,
  updateTreeCliquesAsMarginalizedFromVars!,

  # weiged sampling
  AliasingScalarSampler,
  rand!,
  rand,
  fastnorm,

  # new wrapper (experimental)
  CommonConvWrapper,

  # solve inference
  inferOverTree!,
  inferOverTreeR!,
  inferOverTreeIterative!,

  #development interface
  getTreeCliqSolveOrderUp,
  getCliqOrderUpSolve,
  getCliqVarInitOrderUp,
  getCliqInitVarOrderDown,
  getCliqStatusUp,
  blockCliqUntilChildrenHaveUpStatus,
  blockCliqSiblingsParentNeedDown,
  getCliqNumAssocFactorsPerVar,
  upMsgPassingRecursive,
  downMsgPassingRecursive,

  upMsgPassingIterative!,
  downMsgPassingIterative!,

  # Inference types
  InferenceType,
  PackedInferenceType,
  Singleton,
  Pairwise,
  # introduced for approximate convolution operations
  setThreadModel!,
  SingleThreaded,
  MultiThreaded,

  # functor abstracts
  FunctorInferenceType,
  FunctorPairwise,
  FunctorPairwiseMinimize,
  FunctorSingleton,
  # FunctorPartialSingleton,
  FunctorPairwiseNH,
  FunctorSingletonNH,

  # Solving utils
  findRelatedFromPotential,
  shuffleXAltD!,
  numericRoot,
  numericRootGenericRandomized,
  numericRootGenericRandomizedFnc,
  numericRootGenericRandomizedFnc!,
  solveFactorMeasurements,

  # user functions
  proposalbeliefs,
  predictbelief,
  getCliq,
  getCliqMat,
  getCliqAssocMat,
  getCliqMsgMat,
  getCliqFrontalVarIds,
  getFrontals,                     # duplicate
  getCliqSeparatorVarIds,
  getCliqAllVarIds,
  getCliqVarIdsAll,
  getCliqVars,
  getCliqAllVarSyms,
  getCliqVarIdsPriors,
  getCliqVarSingletons,
  getCliqAllFactIds,
  getCliqFactorIdsAll,
  areCliqVariablesAllMarginalized,
  setTreeCliquesMarginalized!,

  # generic marginal used during elimitation game
  GenericMarginal,
  PackedGenericMarginal,

  uppA,
  convert,
  extractdistribution,

  # factor graph operating system utils (fgos)
  convert2packedfunctionnode,
  encodefg,
  decodefg,
  savejld,
  loadjld,
  landmarks,
  setCliqDrawColor,
  fetchCliqTaskHistoryAll!,
  fetchAssignTaskHistoryAll!,

  # Temp placeholder for evaluating string types to real types
  _evalType,
  saveDFG,
  loadDFG,
  rebuildFactorMetadata!,

  setUpMsg!,
  upMsg,
  getDwnMsgs,
  getUpMsgs,
  setDwnMsg!,
  dwnMsg,
  getDwnMsgs,
  getCliqMsgsUp,
  getCliqMsgsDown,
  getCliqVarSolveOrderUp,

  getSym,
  doCliqInferenceUp!,
  getFactorsAmongVariablesOnly,
  setfreeze!,

  #internal dev functions for recycling cliques on tree
  attemptTreeSimilarClique,
  getCliqSiblingsPartialNeeds,

  # some utils
  compare,
  compareAllSpecial,
  getIdx,
  showFactor,
  showVariable,
  getMeasurements,
  findFactorsBetweenFrom,
  addDownVariableFactors!,
  getDimension,

  # For 1D example,

  # TODO rename to L2 distance
  Ranged,
  PackedRanged,

  # development
  # dev exports
  addGraphsVert!,
  makeAddEdge!,
  shuffleXAltD,
  reshapeVec2Mat, # TODO deprecate

  # OBSOLETE TODO REMOVE #TODO TODO
  subGraphFromVerts,
  getMaxVertId,
  dlapi,
  localapi,
  showcurrentdlapi,
  setdatalayerAPI!,
  DataLayerAPI

# TODO should be deprecated
const NothingUnion{T} = Union{Nothing, T}

# non-free, but not currently use!
include("ccolamd.jl")

# regular
include("FactorGraphTypes.jl")

# const InMemDFGType = DFG.LightDFG{SolverParams} #swap out default in v0.8.0/v0.9.0?
const InMemDFGType = DFG.GraphsDFG{SolverParams} # JT TODO move to somewhere more fitting?

include("BeliefTypes.jl")
include("AliasScalarSampling.jl")
include("DefaultNodeTypes.jl")
include("FactorGraph01.jl")
include("SubGraphFunctions.jl")
include("SerializingDistributions.jl")
include("DispatchPackedConversions.jl")
include("FGOSUtils.jl")
include("CompareUtils.jl")

include("JunctionTreeTypes.jl")
include("JunctionTree.jl")
include("TreeBasedInitialization.jl")
include("GraphConstraintTypes.jl")
include("SolverUtilities.jl")
include("ExplicitDiscreteMarginalizations.jl")
include("InferDimensionUtils.jl")
include("ApproxConv.jl")
include("SolveTree01.jl")
include("CliqStateMachine.jl")
include("CliqStateMachineUtils.jl")

# special variables and factors, see RoME.jl for more examples
include("Variables/Sphere1D.jl")
include("Factors/Sphere1D.jl")

include("AdditionalUtils.jl")
include("SolverAPI.jl")

include("Deprecated.jl")

exportimg(pl) = error("Please do `using Gadfly` before IncrementalInference is used to allow image export.")
function __init__()
    @require Gadfly="c91e804a-d5a3-530f-b6f0-dfbca275c004" begin
      @info "Defining spyCliqMat(..) for visualizing association matrix of a clique in the Bayes (Juntion) tree"

      exportimg(pl) = Gadfly.PNG(pl)

      export spyCliqMat

      """
          $SIGNATURES

      Draw the clique association matrix, with keyword arguments for more or less console print outs.

      Notes
      * Columns are variables, rows are factors.
      * Drawn from up message passing perspective.
      * Blue color implies no factor association.
      * Frontal, separator, and upmessages are all drawn at different intensity of red.
      * Downward messages not shown, as they would just be singletons of the full separator set.
      """
      function spyCliqMat(cliq::Graphs.ExVertex; showmsg=true, suppressprint::Bool=false)
        mat = deepcopy(getCliqMat(cliq, showmsg=showmsg))
        # TODO -- add improved visualization here, iter vs skip
        mat = map(Float64, mat)*2.0.-1.0
        numlcl = size(getCliqAssocMat(cliq),1)
        mat[(numlcl+1):end,:] *= 0.9
        mat[(numlcl+1):end,:] .-= 0.1
        numfrtl1 = floor(Int,length(getData(cliq).frontalIDs) + 1)
        mat[:,numfrtl1:end] *= 0.9
        mat[:,numfrtl1:end] .-= 0.1
        if !suppressprint
          @show getData(cliq).itervarIDs
          @show getData(cliq).directvarIDs
          @show getData(cliq).msgskipIDs
          @show getData(cliq).directFrtlMsgIDs
          @show getData(cliq).directPriorMsgIDs
        end
        sp = Gadfly.spy(mat)
        push!(sp.guides, Gadfly.Guide.title("$(cliq.attributes["label"]) || $(cliq.attributes["data"].frontalIDs) :$(cliq.attributes["data"].conditIDs)"))
        push!(sp.guides, Gadfly.Guide.xlabel("fmcmcs $(cliq.attributes["data"].itervarIDs)"))
        push!(sp.guides, Gadfly.Guide.ylabel("lcl=$(numlcl) || msg=$(size(getCliqMsgMat(cliq),1))" ))
        return sp
      end
      function spyCliqMat(bt::BayesTree, lbl::Symbol; showmsg=true, suppressprint::Bool=false)
        spyCliqMat(whichCliq(bt,lbl), showmsg=showmsg, suppressprint=suppressprint)
      end
    end
end

# Old code that might be used again
# function getType(typestring::AS) where {AS <: AbstractString}
#  # eval(Meta.parse(typestring))()
#  # getfield(Main, Symbol(typestring))
#  getfield(@__MODULE__, Symbol(typestring))
# end

export setSerializationNamespace!, getSerializationModule, getSerializationModules

end


using OpenType

const FONT_LOAD_DIR = joinpath(dirname(dirname((@__DIR__))), "assets", "fonts")

@memoize LRU(maxsize=50) function load_font(font_name::AbstractString)
    for file in readdir(FONT_LOAD_DIR)
        if first(splitext(basename(file))) == font_name
            return OpenTypeFont(joinpath(FONT_LOAD_DIR, file))
        end
    end
    error("Could not find a font file for $font_name.")
end

abstract type TextWidget <: Widget end

struct GlyphCurveIndexing
    curves::Vector{Point{3, Point{2, Float32}}}
    curve_ranges::Vector{UnitRange{Int}}
end

function GlyphCurveIndexing(font::OpenTypeFont)
    ranges = UnitRange{Int}[]
    stop = 0
    cs = map(filter(!isnothing, font.glyphs)) do glyph
        glyph_curves = curves(glyph)
        start = stop + 1
        stop = start + 3 * length(glyph_curves)
        push!(ranges, start:stop)
        glyph_curves
    end
    GlyphCurveIndexing([cs...;], ranges)
end

struct TextRenderInfo
    color::RGBA
    size::Float64
end

struct TextProperties
    font::OpenTypeFont
    render_info::TextRenderInfo
    indexing::GlyphCurveIndexing
end

TextProperties(font::OpenTypeFont; render_info = TextRenderInfo(RGBA(1., 1., 1., 1.), 14.)) = TextProperties(font, render_info, GlyphCurveIndexing(font))

Base.@kwdef struct StaticText <: TextWidget
    text::String
    origin::Point2f
    properties::TextProperties = TextProperties(load_font("juliamono-regular"))
end

"""
Vertex data for a character to be rendered.
"""
struct CharData
    "Vertex position."
    position::Point2f
    "Character-dependent offset index within the vector of all concatenated glyph curves."
    offset::UInt32
    "Character-dependent number of curves."
    count::UInt32
end

vertex_data_type(::Type{StaticText}) = CharData
mesh_encoding_type(T::Type{StaticText}) = MeshVertexEncoding{TriangleList, vertex_data_type(T)}

function GeometryExperiments.PointSet(w::StaticText, char::Char)
    glyph = w.properties.font[char]
    if isnothing(glyph)
        PointSet(Point{2,Float64}[])
    else
        Translation(w.origin)(boundingelement(glyph))
    end
end

function GeometryExperiments.boundingelement(w::StaticText)
    points = [map(char -> PointSet(w, char).points, collect(w.text))...;]
    boundingelement(PointSet(points))
end

function GeometryExperiments.MeshVertexEncoding(w::StaticText)
    T = vertex_data_type(w)
    MT = mesh_encoding_type(w)
    mesh = MT(TriangleList{3,UInt32}([]), T[])

    foreach(enumerate(w.text)) do (i, char)
        glyph = w.properties.font[char]
        idx = findfirst(==(glyph), w.properties.font.glyphs)
        range = w.properties.indexing.curve_ranges[idx]
        offset = range.start - 1
        count = length(range)
        set = PointSet(w, char)
        append!(mesh.encoding.indices, collect(TriangleList(TriangleStrip(1 + 4 * (i - 1):4i))))
        append!(mesh.vertex_data, T.(set.points, offset, count))
    end
    mesh
end

resource_types(::Type{StaticText}) = (GPUResource{Buffer,DeviceMemory,Nothing},)


function _extract_reaction_phrase_table(phrase::String)::Dict{String,Float64}

    # initialize -
    reaction_phrase_table = Dict{String,Float64}()

    # compnents -
    if (occursin("+",phrase) == true)

        # ok, so we have some +'s
        component_array = split(phrase,"+")
        for component in component_array

            # split around the * -
            inner_component_array = split(component,"*")

            # if len(inner_component_array) == 1, then no *
            # if len(inner_component_array) == 2, then we have a st coeff -
            if (length(inner_component_array) == 1)

                # we have a single species -
                key = string(last(inner_component_array))
                value = 1.0
                reaction_phrase_table[key] = value

            elseif (length(inner_component_array) == 2)

                # we have a species w/a coefficient -
                key = string(last(inner_component_array))
                value = parse(Float64, first(inner_component_array))
                reaction_phrase_table[key] = value
            else
                # error state -
                throw(DimensionMismatch("component array larger than 2"))
            end
        end
    else

        # single species, possibly w/*
        # split around the * -
        phrase_component_array = split(phrase,"*")
        if (length(phrase_component_array) == 1)

            # we have a single species -
            key = string(last(phrase_component_array))
            value = 1.0
            reaction_phrase_table[key] = value

        elseif (length(phrase_component_array) == 2)

            # we have a species w/a coefficient -
            key = string(last(phrase_component_array))
            value = parse(Float64, first(phrase_component_array))
            reaction_phrase_table[key] = value

        else
            # error state -
            throw(DimensionMismatch("phrase component array larger than 2"))
        end
    end

    # return -
    return reaction_phrase_table
end

function _extract_stoichiometric_coefficient(phrase::String, speciesSymbol::String)::Float64

    # initialize -
    stoichiometric_coefficient = 0.0 # default is 0, species in NOT involved

    # get the reaction phrase table -
    reaction_phrase_table = _extract_reaction_phrase_table(phrase)

    # ok, look up the species in the phrase table, if its there we have a match - no = 0
    if (haskey(reaction_phrase_table,speciesSymbol) == true)
        return reaction_phrase_table[speciesSymbol]
    end

    # return -
    return stoichiometric_coefficient
end

# == PUBLIC METHODS BELOW HERE ======================================================================================== #

"""
    generate_stoichiometric_matrix(intermediate_dictionary::Dict{String,Any})::VLResult

Generate a stoichiometric matrix based on the biochemical model reaction network.

Input arguments:
`intermediate_dictionary::Dict{String,Any}` - data dictionary containing the master reaction table and molecular species participating in the reactions.

Output arguments:
`VLResult::VLResult` - concrete data type holding the generated stoichiometric matrix.

"""
function generate_stoichiometric_matrix(intermediate_dictionary::Dict{String,Any})::VLResult

    # initialize -

    try

        # check: do we have the ir_master_reaction_table_key?
        if (haskey(intermediate_dictionary, ir_master_reaction_table_key) == false)
            throw(ArgumentError("intermediate_dictionary is missing the ir_master_reaction_table_key"))
        end

        # check: do we have the ir_list_of_molecular_species_key?
        if (haskey(intermediate_dictionary, ir_list_of_molecular_species_key) == false)
            throw(ArgumentError("intermediate_dictionary is missing the ir_list_of_molecular_species_key"))
        end

        # get reaction data frame, and species list -
        reaction_data_frame = intermediate_dictionary[ir_master_reaction_table_key]
        list_of_species = intermediate_dictionary[ir_list_of_molecular_species_key]

        # ok, so we need to initialize some space for the stm -
        number_of_species = length(list_of_species)
        number_of_reactions = size(reaction_data_frame,1)
        stoichiometric_matrix = Array{Float64,2}(undef, number_of_species, number_of_reactions)

        # process each species -
        for (species_index,species_symbol) in enumerate(list_of_species)

            # process each reaction -
            for reaction_index = 1:number_of_reactions

                # get the left and right phrases -
                left_phrase = reaction_data_frame[reaction_index,:left_phrase]
                right_phrase = reaction_data_frame[reaction_index,:right_phrase]

                # check: is this species symbol in the left or right phrase?
                left_st_coeff = _extract_stoichiometric_coefficient(left_phrase, species_symbol)
                right_st_coeff = _extract_stoichiometric_coefficient(right_phrase, species_symbol)

                # overall coeff -
                overall_st_coeff = right_st_coeff - left_st_coeff

                # grab -
                stoichiometric_matrix[species_index,reaction_index] = overall_st_coeff
            end
        end

        # return -
        return VLResult(stoichiometric_matrix)
    catch error
        return VLResult(error)
    end
end

# == PUBLIC METHODS ABOVE HERE ======================================================================================== #


#=
This file is auto-generated. Do not edit.
=#
"""
    mutable struct AVRSimple <: AVR
        Kv::Float64
        ext::Dict{String, Any}
        states::Vector{Symbol}
        n_states::Int64
        internal::InfrastructureSystemsInternal
    end

Parameters of a simple proportional AVR in the derivative of EMF
i.e. an integrator controller on EMF

# Arguments
- `Kv::Float64`: Proportional Gain, validation range: (0, nothing)
- `ext::Dict{String, Any}`
- `states::Vector{Symbol}`
- `n_states::Int64`: Fixed AVR has 1 states
- `internal::InfrastructureSystemsInternal`: power system internal reference, do not modify
"""
mutable struct AVRSimple <: AVR
    "Proportional Gain"
    Kv::Float64
    ext::Dict{String, Any}
    states::Vector{Symbol}
    "Fixed AVR has 1 states"
    n_states::Int64
    "power system internal reference, do not modify"
    internal::InfrastructureSystemsInternal
end

function AVRSimple(Kv, ext=Dict{String, Any}(), )
    AVRSimple(Kv, ext, [:Vf], 1, InfrastructureSystemsInternal(), )
end

function AVRSimple(; Kv, ext=Dict{String, Any}(), )
    AVRSimple(Kv, ext, )
end

# Constructor for demo purposes; non-functional.
function AVRSimple(::Nothing)
    AVRSimple(;
        Kv=0,
        ext=Dict{String, Any}(),
    )
end

"""Get AVRSimple Kv."""
get_Kv(value::AVRSimple) = value.Kv
"""Get AVRSimple ext."""
get_ext(value::AVRSimple) = value.ext
"""Get AVRSimple states."""
get_states(value::AVRSimple) = value.states
"""Get AVRSimple n_states."""
get_n_states(value::AVRSimple) = value.n_states
"""Get AVRSimple internal."""
get_internal(value::AVRSimple) = value.internal


#This file executes the TD3 algorithm: https://arxiv.org/pdf/1802.09477.pdf
#This is more advanced than the simple DDPG algorithm!!

include("src/init.jl")# Include all relevant functions

include("src/td3/episodes.jl")
include("src/td3/gradients.jl")
include("src/td3/updates.jl")

res  = zeros(params.n_iter)
prec = zeros(params.n_iter)

#=NIPS-challenge
include("src/NIPS2018.jl")
#Global Variable of Environment!
@everywhere @pyimport osim.env as osim
@everywhere env = osim.ProstheticsEnv(visualize=false)
env = osim.ProstheticsEnv(visualize=true)
@everywhere env[:change_model](model="2D", prosthetic=false, difficulty=0)
sleep(1)
# =#

#1. Randomly Initialize
#-------------------------------------------------------------
#1.1 First Critic
Qf_1         = init_θ_single_norm(params.structure_net_qf)
Qf_tar_1     = copy(Qf_1)
opt_para_qf1 = optimizers(Qf_1, Adam; lr = 0.5*params.lr_qf)

#1.1 Second Critic
Qf_2         = init_θ_single_norm(params.structure_net_qf2)
Qf_tar_2     = copy(Qf_2)
opt_para_qf2 = optimizers(Qf_2, Adam; lr = 2.0*params.lr_qf)

#1.3 Policy network
Pl           = init_θ_single_norm(params.structure_net_pl)
Pl_tar       = copy(Pl)
opt_para_pl  = optimizers(Pl, Adam; lr = params.lr_pl)

#1.4 Replay Buffer and Latest Trajectories
buffer           = Traj_Data()
buffer.traj      = Array{Trajectory,1}(undef,0)
buffer.timesteps = 0
buffer.n_traj    = 0

#1.5 Buffer for the most recent trajectories
all_traj  = init_all_traj(params)

#using FileIO
#init_nets = load("Results/Videos/10_20_init_nets/mountaincar.jld2")
#Pl = init_nets["θa"]

#main function running DDPG
function TD3()
    weights_old = zeros(length(net_to_vec(Pl)),2)
    a_t_old = all_traj.traj[1].a_t

    #1.4 Init running mean std
    rms_all = init_rms(params.s_init)

    #Iterate
    for i_iter = 1:params.n_iter
        #Init Progress bar
        progress = Progress(params.n_iter,1)

        if floor(i_iter/100) == ceil(i_iter/100)
            #make_video(all_traj,params, i_iter)
            save_data(res, prec, Pl, Qf_1, all_traj, i_iter)
        end

        #2.Collect Data using current stochastic Policy
        #-------------------------------------------------------------
        res[i_iter] = parallel_episodes_td3(params, Pl, Qf_1, Qf_2, all_traj,rms_all)
        #res[i_iter], dump = parallel_episodes_skeleton(params,Pl,Pl_tar,all_traj,rms_all)

        #3.Pump data in Replay Buffer:
        #-------------------------------------------------------------
        #3.1 Generate minibatches from current data_batch
        mini_batch = build_minibatches(all_traj,params)
        #3.2 Pump the data into the buffer
        pump_buffer!(buffer,mini_batch,params)

        #4.Train all the stuff now:
        #-------------------------------------------------------------
        do_td3_update_parallel!(Qf_1, Qf_tar_1, Qf_2, Qf_tar_2, Pl, Pl_tar, buffer, params, opt_para_pl, opt_para_qf1, opt_para_qf2)

        #5.Visualize Progress
        ProgressMeter.update!(progress,i_iter) #update progress bar

        #6. Plot data of interest
        if floor(i_iter/5) == ceil(i_iter/5)
            print("  new visualization: ")
            a_t_old ,weights_old = plot_stuff(params, all_traj, a_t_old,
            weights_old, i_iter,Pl, Qf_1, res, prec/params.n_traj, Pl, Pl)
            #run_episode_skeleton_demo!(params,Pl,Pl_tar,all_traj.traj[1],rms_all,env)
        end
        print("  Result:  ", res[i_iter])
    end
end

TD3()


module GnssWrapper

using JavaCall

const AbsoluteDate = @jimport org.orekit.time.AbsoluteDate
const BeidouAlmanac = @jimport org.orekit.gnss.BeidouAlmanac
const BeidouOrbitalElements = @jimport org.orekit.propagation.analytical.gnss.BeidouOrbitalElements
const Class = @jimport java.lang.Class
const CombinationType = @jimport org.orekit.estimation.measurements.gnss.CombinationType
const CombinedObservationData = @jimport org.orekit.gnss.CombinedObservationData
const CombinedObservationDataSet = @jimport org.orekit.gnss.CombinedObservationDataSet
const DOP = @jimport org.orekit.gnss.DOP
const DOPComputer = @jimport org.orekit.gnss.DOPComputer
const DataProvidersManager = @jimport org.orekit.data.DataProvidersManager
const ElevationMask = @jimport org.orekit.utils.ElevationMask
const Enum = @jimport java.lang.Enum
const Frequency = @jimport org.orekit.gnss.Frequency
const GLONASSAlmanac = @jimport org.orekit.gnss.GLONASSAlmanac
const GLONASSEphemeris = @jimport org.orekit.gnss.GLONASSEphemeris
const GLONASSOrbitalElements = @jimport org.orekit.propagation.analytical.gnss.GLONASSOrbitalElements
const GNSSOrbitalElements = @jimport org.orekit.propagation.analytical.gnss.GNSSOrbitalElements
const GPSAlmanac = @jimport org.orekit.gnss.GPSAlmanac
const GPSOrbitalElements = @jimport org.orekit.propagation.analytical.gnss.GPSOrbitalElements
const GalileoAlmanac = @jimport org.orekit.gnss.GalileoAlmanac
const GalileoOrbitalElements = @jimport org.orekit.propagation.analytical.gnss.GalileoOrbitalElements
const GeodeticPoint = @jimport org.orekit.bodies.GeodeticPoint
const HatanakaCompressFilter = @jimport org.orekit.gnss.HatanakaCompressFilter
const IRNSSAlmanac = @jimport org.orekit.gnss.IRNSSAlmanac
const IRNSSOrbitalElements = @jimport org.orekit.propagation.analytical.gnss.IRNSSOrbitalElements
const InputStream = @jimport java.io.InputStream
const JString = @jimport java.lang.String
const List = @jimport java.util.List
const MeasurementType = @jimport org.orekit.gnss.MeasurementType
const NamedData = @jimport org.orekit.data.NamedData
const Object = @jimport java.lang.Object
const ObservationData = @jimport org.orekit.gnss.ObservationData
const ObservationDataSet = @jimport org.orekit.gnss.ObservationDataSet
const ObservationType = @jimport org.orekit.gnss.ObservationType
const OneAxisEllipsoid = @jimport org.orekit.bodies.OneAxisEllipsoid
const Optional = @jimport java.util.Optional
const QZSSAlmanac = @jimport org.orekit.gnss.QZSSAlmanac
const QZSSOrbitalElements = @jimport org.orekit.propagation.analytical.gnss.QZSSOrbitalElements
const RinexHeader = @jimport org.orekit.gnss.RinexHeader
const RinexLoader = @jimport org.orekit.gnss.RinexLoader
const SEMParser = @jimport org.orekit.gnss.SEMParser
const SatelliteSystem = @jimport org.orekit.gnss.SatelliteSystem
const SignalCode = @jimport org.orekit.gnss.SignalCode
const TimeScale = @jimport org.orekit.time.TimeScale
const TimeScales = @jimport org.orekit.time.TimeScales
const Vector2D = @jimport org.hipparchus.geometry.euclidean.twod.Vector2D
const Vector3D = @jimport org.hipparchus.geometry.euclidean.threed.Vector3D
const YUMAParser = @jimport org.orekit.gnss.YUMAParser

export BeidouAlmanac
export CombinedObservationData
export CombinedObservationDataSet
export DOP
export DOPComputer
export Frequency
export GLONASSAlmanac
export GLONASSEphemeris
export GPSAlmanac
export GalileoAlmanac
export HatanakaCompressFilter
export IRNSSAlmanac
export MeasurementType
export ObservationData
export ObservationDataSet
export ObservationType
export QZSSAlmanac
export RinexHeader
export RinexLoader
export SEMParser
export SatelliteSystem
export SignalCode
export YUMAParser
export compare_to
export compute
export create
export describe_constable
export equals
export filter
export get_af0
export get_af1
export get_af2
export get_agency_name
export get_almanacs
export get_antenna_azimuth
export get_antenna_b_sight
export get_antenna_height
export get_antenna_number
export get_antenna_phase_center
export get_antenna_reference_point
export get_antenna_type
export get_antenna_zero_direction
export get_aodc
export get_aode
export get_approx_pos
export get_bgd
export get_bgde1e5a
export get_bgde5b_e1
export get_center_mass
export get_cic
export get_cis
export get_class
export get_clk_offset
export get_combination_type
export get_combined_m_hz_frequency
export get_crc
export get_crs
export get_cuc
export get_cus
export get_date
export get_declaring_class
export get_delta_i
export get_delta_t
export get_delta_t_dot
export get_e
export get_eccentricities
export get_elevation_mask
export get_frequency
export get_frequency_channel
export get_gamma_n
export get_gdop
export get_glo2utc
export get_glo_offset
export get_gnss_nb
export get_gps2_glo
export get_hdop
export get_header
export get_health
export get_health_e1
export get_health_e5a
export get_health_e5b
export get_i0
export get_i_dot
export get_interval
export get_iod
export get_iod_nav
export get_iodc
export get_iode
export get_iodec
export get_key
export get_lambda
export get_leap_seconds
export get_leap_seconds_day_num
export get_leap_seconds_future
export get_leap_seconds_week_num
export get_list_applied_dcbs
export get_list_applied_pcvs
export get_location
export get_loss_of_lock_indicator
export get_m0
export get_m_hz_frequency
export get_marker_name
export get_marker_number
export get_marker_type
export get_mean_motion
export get_measurement_type
export get_min_elevation
export get_n4
export get_na
export get_name
export get_observation_code
export get_observation_data
export get_observation_data_sets
export get_observation_type
export get_observer_name
export get_omega0
export get_omega_dot
export get_pa
export get_pdop
export get_phase_shift_corrections
export get_prn
export get_prn_number
export get_prn_numbers
export get_ratio
export get_rcvr_clk_offset
export get_receiver_number
export get_receiver_type
export get_receiver_version
export get_rinex_version
export get_sat_configuration
export get_satellite_system
export get_signal_code
export get_signal_strength
export get_signal_strength_unit
export get_sma
export get_source
export get_supported_names
export get_svn
export get_t_first_obs
export get_t_last_obs
export get_tdop
export get_tgd
export get_tgd1
export get_tgd2
export get_time
export get_tn
export get_toc
export get_ura
export get_used_observation_data
export get_value
export get_vdop
export get_wavelength
export get_week
export get_x
export get_x_dot
export get_x_dot_dot
export get_y
export get_y_dot
export get_y_dot_dot
export get_z
export get_z_dot
export get_z_dot_dot
export hash_code
export load_data
export name
export notify
export notify_all
export ordinal
export parse_satellite_system
export still_accepts_data
export to_string
export value_of
export values
export wait
export with_elevation_mask
export with_min_elevation

include("AntennaWrapper/AntennaWrapper.jl")
include("AttitudeWrapper/AttitudeWrapper.jl")
include("beidou_almanac.jl")
include("combined_observation_data.jl")
include("combined_observation_data_set.jl")
include("dop")
include("dop_computer.jl")
include("frequency.jl")
include("galileo_almanac.jl")
include("glonass_almanac.jl")
include("glonass_ephemeris.jl")
include("gps_almanac.jl")
include("hatanaka_compress_filter.jl")
include("irnss_almanac.jl")
include("measurement_type.jl")
include("observation_data.jl")
include("observation_data_set.jl")
include("observation_type.jl")
include("qzss_almanac.jl")
include("rinex_header.jl")
include("rinex_loader.jl")
include("satellite_system.jl")
include("sem_parser.jl")
include("signal_code.jl")
include("yuma_parser.jl")

end


module SurfaceReactions
using LightXML
using DifferentialEquations
include("Constants.jl")
include("Reactions.jl")
include("Utils.jl")
include("IdealGas.jl")

"""
Function to create composite type of SurfaceMechanism and
SpeciesRxnMap. In general the application programs shall only
call the compile_mech function and not directly the create_mech
and species_rxn_map function
#   Usage
    compile_mech(file_path,gas_species)
-   file_path::String : path to the mechanism xml file
-   gas_species::Array{String,1} : list of gasphase species
"""
function compile_mech(file_path::T, thermo_obj::IdealGas.SpeciesThermoObj, gas_species::Array{T,1}) where T <: AbstractString
    sm = create_mech(file_path,gas_species,thermo_obj.molwt)
    rxn_map = species_rxn_map(gas_species,sm)
    return MechanismDefinition(sm,rxn_map)
end

"""
Function to read a surface reaction mechanism file
#   Usage:
    create_mech(file_path,gas_species)
-   file_path::String : file name including the relative path
-   gas_species::Array{String,1} : gasphase species list
"""
function create_mech(file_path::String, gas_species::Array{String,1},molwt::Array{Float64,1})
    xmldoc = parse_file(file_path)
    mech_root = root(xmldoc)
    mech_unit = attribute(mech_root,"unit")
    energy_factor = convert2si(mech_unit)

    #get the surface species
    surface_species = map(x->uppercase(x) ,split(content(get_elements_by_tagname(mech_root,"species")[1])))

    #default initial values
    site_coordination_vector = ones(length(surface_species))
    covg_vec = zeros(length(surface_species))
    site_density = 1.0
    site_name = ""
    #read the site node
    site_node = get_elements_by_tagname(mech_root,"site")
    for elements in site_node
        site_name = attribute(elements,"name")
        #get the site coordination
        coordination = get_elements_by_tagname(elements,"coordination")[1]
        if is_elementnode(coordination)
            site_coord_species = split(content(coordination),",")
            for items in site_coord_species
                sp,val = split(items,"=")
                sp_id = get_index(String(sp),surface_species)
                site_coordination_vector[sp_id] = parse(Float64,String(val))
            end
        end
        #get the site density
        density = get_elements_by_tagname(elements,"density")[1]
        if is_elementnode(density)
            unit = strip(attribute(density,"unit"))
            site_density = parse(Float64, content(density))
            if uppercase(unit) == "MOL/M2"
                site_density *= 1e4
            end
        else
            throw(error("Surface site density not specified\n"))
        end

        #get the intitial coverages
        covg_node = get_elements_by_tagname(elements,"initial")[1]
        if is_elementnode(covg_node)
            ini_covg = split(content(covg_node),",")
            for items in ini_covg
                sp,val = split(items,"=")
                sp_id = get_index(String(sp),surface_species)
                covg_vec[sp_id] = parse(Float64,String(val))
            end
        else
            throw(error("Initial coverage not specified\n"))
        end

    end

    site_info = SiteInfo(site_name,site_density,site_coordination_vector,covg_vec)

    #Read the coverage dependencies if any. This must be read before reading the reactions
    covg_nodes = get_elements_by_tagname(mech_root,"coverage")
    cov_dep_rxns = Dict{Int64,Dict{Int,Float64}}() #rxn_id=>{species_id=>cov_act_energy}
    for covg in covg_nodes
        rxn_ids = split(attribute(covg,"id"))
        sp,val = split(content(covg),"=")
        sp_id = get_index(String(sp),surface_species)
        cov_dict = Dict(sp_id=>parse(Float64,val))

        for rxn in rxn_ids
           cov_dep_rxns[parse(Int64,rxn)] = cov_dict
        end
    end
    #Read the order dependencies if any. This must be read before reading the reactions
    order_nodes = get_elements_by_tagname(mech_root,"order")
    order_dep_rxns = Dict{Int64,Dict{Int,Float64}}() #rxn_id=>{species_id=>order}
    for order in order_nodes
        rxn_ids = split(attribute(order,"id"))
        sp,val = split(content(order),"=")
        sp_id = get_index(String(sp),surface_species)
        order_dict = Dict(sp_id=>parse(Float64,val))
        for rxn in rxn_ids
            order_dep_rxns[parse(Int64,rxn)] = order_dict
        end
    end

    #Read Motz-Wise Correction
    mwc_node = get_elements_by_tagname(mech_root,"mwc")
    mwc_rxns = Array{Int64,1}()
    if length(mwc_node) > 0
        if is_elementnode(mwc_node[1])
            mwc_rxns = [ parse(Int64,rxn) for rxn in split(content(mwc_node[1]))]
        end
    end

    #Define the reaction array to be returned
    rxn_array= Array{ElementaryReactions,1}()

    rct_ids = Array{Int64,1}()
    prdt_ids = Array{Int64,1}()
    all_species = Array{String,1}()
    all_species = copy(gas_species)
    append!(all_species,surface_species)

    #Read sticking reactions
    site_index = get_index(site_name,all_species)

    stick_node = get_elements_by_tagname(mech_root,"stick")[1]
    if is_elementnode(stick_node)
        stick_rxns = get_elements_by_tagname(stick_node,"rxn")
        for rxn in stick_rxns
            rxn_id = parse(Int64,strip(attribute(rxn,"id")))
            stoic, params = split(content(rxn),"@")
            reversible, r_species, p_species = parase_rxn_string(String(stoic))
            rct_ids = [get_index(sp,all_species) for sp in r_species]
            prdt_ids = [get_index(sp,all_species) for sp in p_species]
            s,β,E = parse_rxn_params(String(params))
            #Check for coverage, order and mwc dependency
            cov,ord,mwc = get_dependencies(cov_dep_rxns,order_dep_rxns,mwc_rxns,rxn_id)
            #convert sticking coefficient to rate constant
            n_site = count(x->x==site_index, rct_ids)
            gas_species_id = filter(x->x != site_index, rct_ids)[1]
            sqrt_term = 100*sqrt(R/(2*π*molwt[gas_species_id]))
            s_by_gamma = s/site_density^n_site
            k = sqrt_term*s_by_gamma #rate constant
            #Apply the MWC
            if mwc
                k /= (1-0.5*s)
            end
            p = Parameters(s,k,β,E)

            stoic = Stoichiometry(reversible,rct_ids,prdt_ids,cov,ord)
            elm_rxn = ElementaryReactions(rxn_id,stick,p,mwc,stoic)
            push!(rxn_array,elm_rxn)

        end
    end

    #Read Arrhenius reactions
    arrhenius_node = get_elements_by_tagname(mech_root,"arrhenius")[1]
    if is_elementnode(arrhenius_node)
        arrhenius_rxns = get_elements_by_tagname(arrhenius_node,"rxn")
        for rxn in arrhenius_rxns
            rxn_id = parse(Int64,strip(attribute(rxn,"id")))
            stoic,params = split(content(rxn),"@")
            reversible, r_species, p_species = parase_rxn_string(String(stoic))
            rct_ids = [get_index(sp,all_species) for sp in r_species]
            prdt_ids = [get_index(sp,all_species) for sp in p_species]
            k,β,E = parse_rxn_params(String(params))
            p = Parameters(0.0,k,β,E)
            #Check for coverage, order and mwc dependency. MWC not applicable to Arrhenius type
            cov,ord,mwc = get_dependencies(cov_dep_rxns,order_dep_rxns,mwc_rxns,rxn_id)
            stoic = Stoichiometry(reversible,rct_ids,prdt_ids,cov,ord)
            elm_rxn = ElementaryReactions(rxn_id,arrhenius,p,mwc,stoic)
            push!(rxn_array,elm_rxn)
        end
    end

    return SurfaceMechanism(energy_factor,site_info,surface_species,rxn_array)
end

"""
Create array of structures of SpeciesRxnMap. The SpeciesRxnMap is a
composit structure of species_id in participating reactions and the
stoichiometric coefficients
#   Usage
    species_rxn_map(gas_species,sm)
-   gas_species::Array{String,1} :  list of gas phase species
-   sm::SurfaceMechanism : SurfaceMechanism composite type
"""
function species_rxn_map(gas_species::Array{String,1},sm::SurfaceMechanism)
    #Array of SpeciesRxnMap
    srm_array = Array{SpeciesRxnMap,1}()
    #create list of all species
    all_species = Array{String,1}()
    all_species = copy(gas_species)
    append!(all_species,sm.species)
    #loop over all species
    for sp in all_species
        sp_id = get_index(sp,all_species)
        #check if the species is present in the reactant_ids of a reaction
        rxns = Array{Int64,1}()
        stc = Array{Int64,1}()
        for rxn in sm.reactions
            #count the number of occurances of species in reactants list (using ids)
            n = count(x->x==sp_id,rxn.rxn_stoic.reactant_ids)
            if n > 0
                push!(rxns,rxn.id)
                push!(stc,-n)
            end
            #count the number of occurances of species in the products list (using ids)
            n = count(x->x==sp_id,rxn.rxn_stoic.product_ids)
            if n > 0
                push!(rxns,rxn.id)
                push!(stc,n)
            end
        end
        push!(srm_array,SpeciesRxnMap(sp_id,rxns,stc))
    end
    return srm_array
end

"""
Function to calculate the molar production rate of gas-phase and
surface species based on the surface reaction mechanism. The function
will alter the field s_rate after the calculation.
#   Usage:
    calculate_molar_production_rate!(state,md)
-   state::State : state of the reaction, which is of the composite type State
-   thermo_obj::SpeciesThermoObj : Composite type SpeciesThermoObj. Required in case of reversible reactions
-   md::MechanismDefinition : the mechanism, which is of the composite type MechanismDefinition
"""
function calculate_molar_production_rates!(state::ReactionState, thermo_obj::IdealGas.SpeciesThermoObj,md::SurfaceMechDefinition)
    #get the mole fractions from state
    mf = state.mole_frac
    #convert mole fractions to concentrations
    all_conc = (state.p/R/state.T) .* mf
    all_conc *= 1e-6 # convert to mol/cm3
    #get the surface coverages from state
    coverage = state.covg
    #convert coverages to surface concentrations mol/cm2
    surf_conc =  (md.sm.si.density .* coverage) ./ md.sm.si.site_coordination
    #merge the concentrations
    append!(all_conc,surf_conc)
    rxn_rate = Array{Float64,1}()

    #calcuate the rate of individual reactions
    for rxn in md.sm.reactions
        local prdt_conc = prod(all_conc[rxn.rxn_stoic.reactant_ids])
        #check for coverage dependency of the reaction
        local cov_act_E = 0
        #cov_act_E = sum(coverage[collect(keys(rxn.rxn_stoic.cov_dep))] .* collect(values(rxn.rxn_stoic.cov_dep)))
        if !isempty(rxn.rxn_stoic.cov_dep)
            #calculate coverage dependent activation energy
            for (sp_id,cov_e) in rxn.rxn_stoic.cov_dep
                cov_act_E += coverage[sp_id]*cov_e
            end
        end
        #check for order dependency
        local m_order = 1.0
        if !isempty(rxn.rxn_stoic.order_dep)
            #calculate order change
            for (sp_id,order) in rxn.rxn_stoic.order_dep
                m_order *= coverage[sp_id]^order
            end
        end
        #calculate the forward reaction rate
        E = (rxn.params.E + cov_act_E)*md.sm.energy_factor
        k = rxn.params.k0 * state.T^rxn.params.β * exp(-E/R/state.T)
        if rxn.type == stick
            k *= sqrt(state.T)
        end
        push!(rxn_rate,k*prdt_conc*m_order*1e4)#e4 is to convert the units to mol/m2

        if rxn.rxn_stoic.reversible
            throw(error("Reverse reaction not implemented\n"))
        end

    end
    #calculate the species molar production rates
    for srm in md.srm_array
        state.s_rate[srm.species_id] = sum(rxn_rate[srm.rxns] .* srm.stoic_coeff)
    end
end

"""
Function to calculate the molar production rate of gas-phase species.
This function integrates the rate equations so that \frac{d\theta}{dt}=0
#   Usage:
    calculate_molar_production_rate!(state,md)
-   state::State : state of the reaction, which is of the composite type State
-   thermo_obj::SpeciesThermoObj : Composite type SpeciesThermoObj. Required in case of reversible reactions
-   md::MechanismDefinition : the mechanism, which is of the composite type MechanismDefinition
"""
function calculate_ss_molar_production_rates!(state::ReactionState, thermo_obj::IdealGas.SpeciesThermoObj,md::SurfaceMechDefinition, time =1.0)
    t_span = (0,time)
    coverage = state.covg
    p = (state,thermo_obj,md)
    prob = ODEProblem(covg_integration!,coverage,t_span,p)
    sol = solve(prob, alg_hints=[:stiff] , reltol=1e-6, abstol=1e-10, save_everystep=false,save_start=false)
    state.covg = sol.u[lastindex(sol.u)]
    return sol.t, state
end

function covg_integration!(du, u, p, t)
    state, thermo_obj,md = p
    state.covg = u
    ng = length(state.mole_frac)
    calculate_molar_production_rates!(state,thermo_obj,md)
    @inbounds for i in eachindex(u)
        du[i] = (md.sm.si.site_coordination[i]*state.s_rate[ng+i])/(md.sm.si.density*1e4)
    end
end

"""
Function to update the rate parameters. This function is only used for
sensitivity analysis
#   Usage
    update_params(rate_params,md)
-   rate_params::Array  : One dimentional array of floating point variables
-   md:SurfaceMechDefinition : SurfaceMechDefinition struct
"""
function update_params(rate_params::Array{Float64,1},md::SurfaceMechDefinition)
    #assign the new parameters to SurfaceMechDefinition object
    for i in eachindex(rate_params)
        if Int(md.sm.reactions[i].type) == Int(stick)
            k = md.sm.reactions[i].params.k0 #This is the current value of converted rate constant including MWC
            if md.sm.reactions[i].mwc == true
                #take out the MWC factor
                k *= (1-0.5*md.sm.reactions[i].s)
            end
            #new k value for the purturbed sticking coefficient
            k = (k/md.sm.reactions[i].params.s)*rate_params[i]
            md.sm.reactions[i].params.s = rate_params[i]
            if md.sm.reactions[i].mwc ==  true
                #Apply the MWC back
                k /= (1-0.5*rate_params[i])
            end
            md.sm.reactions[i].params.k0 = k
        else
            md.sm.reactions[i].params.k0 = rate_params[i]
        end
    end

end

"""
A function for use in global sensitivity analysis. The function updates the parameters with the
    supplied rate_params
#   Usage
    update_params!(md, rate_params,rxn_ids, constraint_ids)
-   md : SurfaceMechDefinition structure
-   rate_params : parameters to that needs to be set to the mechanism object
-   rxn_ids : id of the reactions considered in the sensitivity analysis
-   constraint_ids : id of the reverse reaction ids considered
"""
function update_params!(md, rate_params::Array{Float64,1},rxn_ids::Array{Int64,1}, constraint_ids::Array{Int64,1}, pratio::Array{Float64,1})
    for id in eachindex(rxn_ids)
        if Int(md.sm.reactions[rxn_ids[id]].type) == Int(stick)
            # The current value of sticking coefficient converted to rate constant including MWC
            k = md.sm.reactions[rxn_ids[id]].params.k0
            if md.sm.reactions[rxn_ids[id]].mwc == true
                #take out the MWC factor
                k *= (1-0.5*md.sm.reactions[rxn_ids[id]].params.s)
            end
            # multiplication factor used to convert sticking coefficient to rate constant
            m_factor = k/md.sm.reactions[rxn_ids[id]].params.s
            # new sticking coefficient
            md.sm.reactions[rxn_ids[id]].params.s = rate_params[id]
            #new k value for the purturbed sticking coefficient
            k  = md.sm.reactions[rxn_ids[id]].params.s * m_factor
            #Apply the MWC back
            if md.sm.reactions[id].mwc == true
                k /= (1-0.5*rate_params[id])
            end
            md.sm.reactions[rxn_ids[id]].params.k0 = k

            # Now if there is a constraint, i.e. if a reverse reaction present, then the
            # parameter for that reaction needs to be changed accordingly.
            if constraint_ids[id] != 0
                md.sm.reactions[constraint_ids[id]].params.k0 = k/pratio[id]
            end
        else
            md.sm.reactions[rxn_ids[id]].params.k0 = rate_params[id]
            if constraint_ids[id] != 0
                md.sm.reactions[constraint_ids[id]].params.k0 = rate_params[id]/pratio[id]
            end
        end
    end
end

#end of module
end