using DataStructures, DataArrays , DataFrames, StatsFuns, GLM , Distributions, MixedModels, StatsBase, StatLib, JSON, NamedArrays

const cfgDefaults=OrderedDict( :P2_Competitor => true
                        ,:pvalue_lvl => 0.20  #pvalue_lvl = 0.20
                        ,:excludedBreaks => String[]    #["estimated_hh_income","hh_age","number_of_children_in_living_un","person_1_gender"]
                        ,:excludedLevels => ["none"]
                        ,:excludedKeys => String[]
                        ,:exposed_flag_var => :exposed_flag
                        ,:sigLevel => "0.2"
                        ,:random_demos => [:estimated_hh_income,:hh_age,:number_of_children_in_living_Un,:person_1_gender]
                        ,:random_campaigns => []
                        ,:dropvars => [:exposed_flag,:dma,:dma_code]
                        ,:scoring_vars => [:Prd_1_Net_Pr_PRE,:Prd_1_Net_Pr_POS,:Buyer_Pos_P0,:Buyer_Pre_P0]
                        ,:occ_y_var => :Trps_POS_P1
                        ,:occ_logvar => :Trps_PRE_P1
                        ,:dolocc_y_var => :Dol_per_Trip_POS_P1
                        ,:dolocc_logvar => :Dol_per_Trip_PRE_P1
                        ,:pen_y_var => :Buyer_Pos_P1
                        ,:pen_logvar => :Buyer_Pre_P1
                        ,:TotalModelsOnly=>false
                       )

"""
# ************ CUSTOMIZE *********************
#jennie7_979
cfgDefaults[:random_demos] = [:estimated_hh_income,:hh_age,:number_of_children_in_living_Un,:person_1_gender]
cfgDefaults[:random_campaigns] = Symbol[:creative,:placement,:publisher]
cfgDefaults[:exposed_flag_var] = :exposed_flag_new
root="/mapr/mapr04p/analytics0001/analytic_users/jmdl/jennie7_979"

# Dict{Any,Any}(Pair{Any,Any}(:random_demos,Symbol[:estimated_hh_income,:hh_age,:number_of_children_in_living_Un,:person_1_gender]),Pair{Any,Any}(:random_campaigns,Symbol[:creative,:placement,:publisher]),Pair{Any,Any}(:root,"/mapr/mapr04p/analytics0001/analytic_users/jmdl/jennie7_979"))
"""

root = !isdefined(:root) ? pwd() : root
cfgDefaults = mergeDict(dict_Sym(read_cfg(root)),cfgDefaults)

# ************ END CUSTOMIZE *********************

# TEst speed improvements
#bigStr(fname::String) = open(fname) do f  return readstring(f) end
#s = bigStr(root*"/orig.csv")
#function loadDF()
#    open(root*"/orig.csv") do f
#       s = readstring(f)
#    end
#    readtable(IOBuffer(s))
#end
#dfdt = loadDF()

cd(root)
function loadDF()
    #cd("/media/u01/analytics/scoring/Healthy/modeling5/")
    #df_data = readtable("csv_final_healthychoice1_I3.csv",header=true); #lowercase(df_data)
    #df_h = readtable("Headers_healthy_choice3.csv",header=false); #lowercase(df_data)
    #names!(df_data, convert(Array{Symbol}, df_h[:x1]) )
    df_data = readtable(root*"/orig.csv",header=false);
    df_h = readtable(root*"/origHead.csv",header=false);
    names!(df_data, convert(Array{Symbol}, df_h[:x1]) )
end
df_in=loadDF()

function Pre_out(df_in::DataFrame)
     df_cat_pre = df_in[df_in[:Buyer_Pre_P0] .==1 , [:Prd_0_Net_Pr_PRE,:experian_id]]
     df_cat_pos = df_in[(df_in[:Buyer_Pre_P0] .==0) & (df_in[:Buyer_Pos_P0] .==1) , [:experian_id]]
     median_df = median(df_cat_pre[:Prd_0_Net_Pr_PRE])
     df_cat_pre[:Prd_0_Net_Pr_PRE_med1] = abs(df_cat_pre[:Prd_0_Net_Pr_PRE]-median_df)
     MAD=median(df_cat_pre[:Prd_0_Net_Pr_PRE_med1])
     df_cat_pre[:Prd_0_Net_Pr_PRE_med2] = (0.6745*(df_cat_pre[:Prd_0_Net_Pr_PRE]-median_df))/MAD
     df_cat_pre_zsc = df_cat_pre[abs(df_cat_pre[:Prd_0_Net_Pr_PRE_med2]) .< 3.5,:]
     df_cat_pre_zsc_1 = df_cat_pre_zsc[:,[:experian_id]]
     df_cat_pre_zsc_f = vcat(df_cat_pos,df_cat_pre_zsc_1)
     df_in_pout =  join(df_in, df_cat_pre_zsc_f, on =  :experian_id , kind = :inner);
end
df_in = Pre_out(df_in);

function isValid(df_data::DataFrame,cfg::OrderedDict)
    function checkValid(iarr::Array{Symbol})  length(setdiff(iarr, names(df_data))) > 0 ? false : true end
    !checkValid(cfg[:all_mandatory_vars]) ? error("ERROR: Not all mandatory_vars in dataset ") : println("VALID : mandatory_vars")
    !checkValid(cfg[:scoring_vars]) ? error("ERROR: Not all scoring_vars in dataset ") : println("VALID : scoring_vars")
end

function getCFG(df_in::DataFrame)
    cfg=StatLib.loadCFG(cfgDefaults, pwd()*"/app.cfg")
    #cfg[:exposed_flag_var] = :exposed_flag_new                  # Go In app.cfg
    #cfg[:random_campaigns] = [:Publisher_Fct1,:Targeting_Fct1]  # Go In app.cfg
    cfg[:allrandoms] = vcat(cfg[:random_demos],cfg[:random_campaigns])
    #cfg[:ProScore] = grep1("MODEL",names(df_in))
    ps = filter(x->contains(string(x), "MODEL"), names(df_in)) # Set the ProScore variable in the dataset   #push!(cfg[:all_mandatory_vars], cfg[:ProScore])
    cfg[:ProScore] = length(ps) > 0 ? ps[1] : :MISSING_MODEL_VARIABLE_IN_DATA
    cfg[:num_products] = length( grep("Buyer_Pre_P",names(df_in)))-1  # get number of products in the data
    #rework vars after loading cfg
    cfg[:random_campaigns] = intersect(cfg[:random_campaigns],names(df_in))

    cfg[:occ_logvar_colname] = Symbol("LOG_"*string(cfg[:occ_logvar]))
    cfg[:dolocc_logvar_colname] = Symbol("LOG_"*string(cfg[:dolocc_logvar]))
    cfg[:pen_logvar_colname] = Symbol("LOG_"*string(cfg[:pen_logvar]))

    cfg[:all_mandatory_vars] = vcat( [:experian_id,
                                      cfg[:pen_y_var],cfg[:pen_logvar],
                                      cfg[:occ_y_var], cfg[:occ_logvar],
                                      cfg[:dolocc_y_var], cfg[:dolocc_logvar]
                                     ]
                                     , cfg[:random_demos]
                                   )
    return cfg
end
cfg=getCFG(df_in)

isValid(df_in,cfg)

function reworkCFG!(df_in::DataFrame,cfg::OrderedDict)
    xflags=[cfg[:exposed_flag_var],:exposed_flag,:Nonbuyer_Pre_P1]  # :Nonbuyer_Pre_P1 no longer used
    features = setdiff(names(df_in),xflags)
    if :state in names(df_in)
        cfg[:xVarsDemos]=vcat([:experian_id, :banner, :Prd_0_Qty_PRE],features[findfirst(features, :state):findfirst(features, :Mosaic)])   # exclude demos between....
    else
        cfg[:xVarsDemos]=Symbol[]
    end
    cfg[:xVarsDemos]=setdiff(cfg[:xVarsDemos],[:person_1_gender,:number_of_children_in_living_Un]) # exclude person, # from non-used demos
    cfg[:xVarsPost] = grep(["POS","Pos","Buyer_Pre_","Nonbuyer_Pre_"],features)   #exclude POST variables
    cfg[:iVarsPREPOS] = grep(["PRE_POS","Pre_Pos"],features)
    features=setdiff(features, setdiff(  vcat(cfg[:xVarsDemos],cfg[:xVarsPost]) ,cfg[:iVarsPREPOS])  )
    cfg[:xVarsP0] =  setdiff(grep("0", features ),[cfg[:ProScore]] ) #exclude category variables(exclude P0's)
    features=setdiff(features,cfg[:xVarsP0])
    cfg[:xVarsReports] = grep(["Perc_","Pr_per_"],features)  #exclude Reporting vars
    features=setdiff(features,cfg[:xVarsReports])
    cfg[:xvars] = vcat(xflags,cfg[:xVarsDemos], cfg[:xVarsPost],cfg[:xVarsP0], cfg[:xVarsReports],cfg[:dropvars])
    features=setdiff(features,cfg[:dropvars])
    cfg[:ivars] = vcat(cfg[:iVarsPREPOS],cfg[:all_mandatory_vars],cfg[:scoring_vars])
    cfg[:ALL_vars_to_exclude] = setdiff(cfg[:xvars], cfg[:ivars])
    df_in[:group] = df_in[cfg[:exposed_flag_var]]
    df_in[:panid] = df_in[:experian_id]
    features = setdiff(unique(vcat(features,cfg[:iVarsPREPOS],cfg[:all_mandatory_vars],cfg[:scoring_vars],[:group,:panid])  ) ,[:experian_id] )
    cfg[:negativevars] = grep(vec(hcat([["P"*string(i),string(i)*"_"] for i=3:cfg[:num_products]]...)),features)   # get variables that need to have negative sign
    cfg[:positivevars] = grep(["P1", "1_","MODEL"], features)    # get variables that need to have positive sign
    if cfg[:P2_Competitor] == true
        cfg[:negativevars] = unique(vcat(cfg[:negativevars],grep(["P2","2_"],features)))
    else
        cfg[:positivevars] = unique(vcat(cfg[:positivevars],grep(["P2","2_"],features)))
    end
    return df_in[features]
end

dfd = reworkCFG!(df_in,cfg)

#######################################
#--------- Data Manipulation ---------#
#######################################

function data_Prep(dfd::DataFrame, cfg::OrderedDict)
    dfd[:isO]=false
    dfd[dfd[:number_of_children_in_living_Un].>=4,:number_of_children_in_living_Un] = 4  # aggregate #of children for 4+ L_114
    if typeof(dfd[:group]) in [DataArray{String,1}]
        dfd[ findin(dfd[:group],["//N","\\N"]), :group] = "0"
        dfd[DataArrays.isna(dfd[:group]), :group]="0"
        dfd[:group] = [parse(Int64,s) for s = dfd[:group]]
    else
        dfd[ DataArrays.isnan(dfd[:group]), :group] = 0
    end
    vars = DataFrame(names=names(dfd),eltypes=eltypes(dfd))
    #for c in setdiff(y,cfg[:allrandoms]) # set variables as.numeric and replace NA's with zero
    for c in setdiff(vars[findin(vars[:eltypes],[String]),:names],cfg[:allrandoms])
        print("Convert String: String->Numeric: ",c)
        try dfd[c] = map( x -> DataArrays.isna.(x) ?  NaN : convert(Float64, x)  , dfd[c]) catch e println("  (failed)") end  #NA to NaN
        try dfd[c] = convert(Array{Float64}, dfd[c]) catch e end   # To Float64
    end
    vars = DataFrame(names=names(dfd),eltypes=eltypes(dfd))
    for c in setdiff(vars[findin(vars[:eltypes],[Float64]),:names],cfg[:allrandoms])  # replace NaN's with zero for numeric variables
        println("Replace Float64 NaN (0.0): ",c)
        dfd[ DataArrays.isnan(dfd[c]), c] = 0.0
    end
    for c in  setdiff(vars[findin(vars[:eltypes],[Int64]),:names],cfg[:allrandoms])
        println("Replace Int64 NaN (0) : ",c)
        dfd[ DataArrays.isnan(dfd[c]), c] = 0
    end
    # NOTE : Do QCs here
    dfd[dfd[:person_1_gender].=="U",:isO] = true   # remove HHs with no gender info
    #dfd[dfd[:person_1_gender].=="U",:whyO] = "person_1_gender=U; remove HHs with no gender info"
    dfd[findin(dfd[:estimated_hh_income],["U","L"]),:estimated_hh_income]="L" # aggregate U and L levels of hh income

    for r in cfg[:random_campaigns]    # check and drop exposed HHs with no publisher info or non-exposed HHs with publisher info
        dfd[findin(dfd[r],["\\N","NULL","0","NONE"])  ,r] ="none"
        #dfd[ (dfd[:isO].==false) & (dfd[r].!="none") & (dfd[:group].==0) ,:whyO] = "non exposed HHs with publisher info"
        dfd[ (dfd[:isO].==false) & (dfd[r].!="none") & (dfd[:group].==0) ,:isO] = true
        #println(r," non exposed HHs with publisher info : ",nrow(dfd[dfd[:whyO].=="non exposed HHs with publisher info",:] )   )
        #dfd[ (dfd[:isO].==false) & (dfd[r].=="none") & (dfd[:group].!=0) ,:whyO] = "exposed HHs with no publisher info"
        dfd[ (dfd[:isO].==false) & (dfd[r].=="none") & (dfd[:group].!=0) ,:isO] = true
        #println(r," exposed HHs with no publisher info : ",nrow(dfd[dfd[:whyO].=="exposed HHs with no publisher info",:] )   )
    end
    # segments for outliers detection
    dfd[:data_NB_NE_B] = false
    dfd[ (dfd[:Buyer_Pre_P1].==0 ) & (dfd[:group].==0 ) & (dfd[:Buyer_Pos_P1].==1 ) ,:data_NB_NE_B] = true
    dfd[:data_B_E_NB] = false
    dfd[ (dfd[:Buyer_Pre_P1].==1 ) & (dfd[:group].==0 ) & (dfd[:Buyer_Pos_P1].==0 ) ,:data_B_E_NB] = true
    dfd[:pen_reduction] = false
    dfd[ (dfd[:Buyer_Pre_P1].==1) & (dfd[:group].==0) & (dfd[:Buyer_Pos_P1].==0 )  ,:pen_reduction] = true
    dfd[:occ_reduction] = false
    dfd[ (dfd[:group].==0) & (dfd[:Buyer_Pos_P1].==1) & (dfd[:Trps_POS_P1].< dfd[:Trps_PRE_P1] ) ,:occ_reduction] = true
    dfd[:dolocc_reduction] = false
    dfd[  (dfd[:group].==0) & (dfd[:Buyer_Pos_P1].==1) & (dfd[:Dol_per_Trip_POS_P1].< dfd[:Dol_per_Trip_PRE_P1] )  , :dolocc_reduction] = true
    return dfd[dfd[:isO].==false, : ]   #[setdiff(names(dfd),[:isO,:whyO])]
end

dfd = data_Prep(dfd, cfg);

#function Restrict50Buyers()
#    for (k,v) in Dict(r=>countmap(dfd[(dfd[:Buyer_Pos_P1].==1)&(dfd[:group].==1), r]) for r in cfg[:random_campaigns])
#            lvls=String[]
#            for (kl,vl) in v
#                if vl < 50
#                    println("$k :: $kl to Other")
#                    dfd[dfd[k].==kl,k] = "Other"
#                end
#                #if vl > 50 push!(lvls,kl) end
#            end
#            #toNone = setdiff(levels(dfd[k]),lvls)
#            #println("\n\n\n$k       ::::          ",toNone)
#    end
#end
#Restrict50Buyers()

function Restrict50Buyers()
    for (k,v) in Dict(r=>countmap(dfd[(dfd[:Buyer_Pos_P1].==1), r]) for r in cfg[:random_campaigns])
            lvls=String[]
            for (kl,vl) in v
                if vl < 25
                    println("$k :: $kl to Other")
                    dfd[dfd[k].==kl,k] = "Other"
                end
                #if vl > 50 push!(lvls,kl) end
            end
            #toNone = setdiff(levels(dfd[k]),lvls)
            #println("\n\n\n$k       ::::          ",toNone)
    end
end
Restrict50Buyers()

function MatchMe(dfd::DataFrame,cfg::OrderedDict)
    df=dfd[dfd[:isO].==false,:]
    df_exp     = df[df[:group].==1,:]
    df_unexp   = df[df[:group].==0,:]
    df_exp_dim   = nrow(df_exp)
    df_unexp_dim = nrow(df_unexp)
    new_unexp_dim = df_unexp_dim*(df_unexp_dim>2000000 ? 0.3 : df_unexp_dim>1000000 ? 0.4 : df_unexp_dim>750000 ? 0.6 : 0.7)
    if length(string(cfg[:ProScore])) == 0
        df_unexp_1 =  df[(df[:group].==0)&(df[:Buyer_Pre_P1].==1),:]
        df_unexp_0 =  df[(df[:group].==0)&(df[:Buyer_Pre_P1].==0),:]

        df_exp_1_dim = nrow(df[(df[:group].==1)&(df[:Buyer_Pre_P1].==1),:])
        df_exp_0_dim = nrow(df[(df[:group].==1)&(df[:Buyer_Pre_P1].==0),:])
        df_unexp_1_dim = nrow(df_unexp_1)
        df_unexp_0_dim =  nrow(df_unexp_0)
        dim_sample0 = round(Int64, (new_unexp_dim-df_exp_dim  ) / (1+(df_exp_1_dim / df_exp_0_dim)) )
        dim_sample1 = round(Int64, new_unexp_dim - df_exp_dim - dim_sample0)

        new_df_unexp_1 = df_unexp_1[sample(1:size(df_unexp_1,1), dim_sample1 ),:]
        new_df_unexp_0 = df_unexp_0[sample(1:size(df_unexp_0,1), dim_sample0 ),:]
        dfd_sample  = vcat(df_exp,new_df_unexp_1,new_df_unexp_0)
    elseif length(string(cfg[:ProScore])) > 0
        sample_control_data=similar(df_unexp, 0)
        for (key, value) in countmap(df_exp[cfg[:ProScore]])
            sample_dim=round(Int64,new_unexp_dim*(value/df_exp_dim))
            temp_data = df_unexp[df_unexp[cfg[:ProScore]].==key,:]
            if sample_dim > size(temp_data,1) sample_dim = size(temp_data,1) end  #??
            samp_data = temp_data[sample(1:size(temp_data,1), sample_dim, replace=false),:]
            sample_control_data = vcat(sample_control_data,    samp_data   )
        end

        sample_data = vcat(sample_control_data,df_exp)
        sample_df_unexp = sample_data[sample_data[:group].==0,:]
        sample_df_exp   = sample_data[sample_data[:group].==1,:]
        sample_df_unexp_1 = sample_data[(sample_data[:group].==0)&(sample_data[:Buyer_Pre_P1].==1),:]
        sample_df_unexp_0 = sample_data[(sample_data[:group].==0)&(sample_data[:Buyer_Pre_P1].==0),:]
        sample_df_exp_1 =  sample_data[(sample_data[:group].==1)&(sample_data[:Buyer_Pre_P1].==1) ,:]
        sample_df_exp_0 =  sample_data[(sample_data[:group].==1)&(sample_data[:Buyer_Pre_P1].==0) ,:]
        sample_df_unexp_1_dim = nrow(sample_df_unexp_1)
        sample_df_unexp_0_dim = nrow(sample_df_unexp_0)
        sample_df_exp_1_dim = nrow(sample_df_exp_1)
        sample_df_exp_0_dim = nrow(sample_df_exp_0)
        dim_sampleA = round(Int64,(sample_df_exp_1_dim/sample_df_exp_0_dim)*sample_df_unexp_0_dim)
        dim_sampleB = round(Int64,(sample_df_exp_0_dim/sample_df_exp_1_dim)*sample_df_unexp_1_dim)

        if sample_df_unexp_1_dim/sample_df_unexp_0_dim > sample_df_exp_1_dim/sample_df_exp_0_dim
            new_df_unexp_1 = sample_df_unexp_1[sample(1:sample_df_unexp_1_dim,dim_sampleA , replace=false),:]
            dfd_sample = vcat(sample_df_exp,new_df_unexp_1,sample_df_unexp_0)
        else
            new_df_unexp_0 = sample_df_unexp_0[sample(1:sample_df_unexp_0_dim, dim_sampleB, replace=false ),:]
            dfd_sample = vcat(sample_df_exp,sample_df_unexp_1,new_df_unexp_0)
        end
    end
    rows2remove = setdiff(dfd[dfd[:isO].==false, :panid],dfd_sample[:panid])
    #dfd[findin(dfd[:panid],rows2remove),:whyO]="NoMatch"
    dfd[findin(dfd[:panid],rows2remove),:isO]=true

#    #Matching - pre-post
"""
    dfp=join(by(dfd[(dfd[:isO].==false),:],[:group,:Buyer_Pos_P1], df->DataFrame(cnt=size(df,1))) ,
             by(dfd[(dfd[:isO].==false),:],:group,df->DataFrame(tot=size(df,1)))
             , on=:group
            )
    dfp[:prop] = dfp[:cnt]./dfp[:tot]
    if dfp[(dfp[:group].==1)&(dfp[:Buyer_Pos_P1].==1),:prop][1] < dfp[(dfp[:group].==0)&(dfp[:Buyer_Pos_P1].==1),:prop][1]

        scnt = Int( round(( dfp[(dfp[:group].==0)&(dfp[:Buyer_Pos_P1].==0),:cnt][1]*dfp[(dfp[:group].==1)&(dfp[:Buyer_Pos_P1].==1),:prop][1] ) / (1-dfp[(dfp[:group].==1)&(dfp[:Buyer_Pos_P1].==1),:prop][1])  ))
        smp = sample(dfd[(dfd[:isO].==false)&(dfd[:Buyer_Pos_P1].==1)&(dfd[:group].==0),:panid], scnt , replace=false)

        oliers = setdiff(dfd[(dfd[:isO].==false)&(dfd[:Buyer_Pos_P1].==1)&(dfd[:group].==0),:panid],smp)

        dfd[findin(dfd[:panid], setdiff(dfd[(dfd[:isO].==false)&(dfd[:Buyer_Pos_P1].==1)&(dfd[:group].==0),:panid],smp)),:isO] =true
   end
"""
        # ---- end matching ------------

    return dfd[dfd[:isO].==false, : ]  #[setdiff(names(dfd),[:isO,:whyO])]
end

dfd = MatchMe(dfd,cfg)

lowercase!(dfd)
cfg=lowercase(cfg)

######################################
#------------MODEL OBJECTS-----------#  [:fea_or_dis_trps_shr_dpp_p1,:fea_or_dis_trps_shr_dpp_p2,:fea_or_dis_trps_shr_dpp_p3,:fea_or_dis_trps_shr_dpp_p4]
######################################

iocc = Dict(:modelName=>:occ, :raneff=>cfg[:random_campaigns], :y_var=>:trps_pos_p1, :logvar=>:LOG_trps_pre_p1, :logvarOrig=>:trps_pre_p1 )
idolocc = Dict(:modelName=>:dolocc, :raneff=>cfg[:random_campaigns], :y_var=>:dol_per_trip_pos_p1, :logvar=>:LOG_dol_per_trip_pre_p1, :logvarOrig=>:dol_per_trip_pre_p1)
ipen = Dict(:modelName=>:pen, :raneff=>cfg[:random_campaigns], :y_var=>:buyer_pos_p1, :logvar=>:LOG_buyer_pre_p1, :logvarOrig=>:buyer_pre_p1 )

custom_vars = [:iso,:whyo,:data_nb_ne_b, :data_b_e_nb ,:pen_reduction,:occ_reduction,:dolocc_reduction]
for m in [iocc,idolocc,ipen] dfd[m[:logvar]]=log(Array(dfd[m[:logvarOrig]]+1)) end

function genExcludeVars!(iocc::Dict,idolocc::Dict,ipen::Dict)
    iocc[:exclude_vars] = vcat( custom_vars,
                                [ :buyer_pos_p1, iocc[:logvarOrig]
                                  ,idolocc[:y_var],idolocc[:logvar],idolocc[:logvarOrig]
                                 ,ipen[:y_var],ipen[:logvar],ipen[:logvarOrig]
                               ]
                              )
    idolocc[:exclude_vars] = vcat( custom_vars,
                                   [ :buyer_pos_p1, idolocc[:logvarOrig]
                                           ,iocc[:y_var],iocc[:logvar],iocc[:logvarOrig]
                                           ,ipen[:y_var],ipen[:logvar],ipen[:logvarOrig]
                                   ]
                                 )

    ipen[:exclude_vars] = vcat( custom_vars,
                                [ ipen[:logvarOrig]
                                  ,iocc[:y_var],iocc[:logvar],iocc[:logvarOrig]
                                  ,idolocc[:y_var],idolocc[:logvar],idolocc[:logvarOrig]
                                ]
                                 )
end
genExcludeVars!(iocc,idolocc,ipen)

function featureSelection(dfd::DataFrame, m::Dict)
    function rmVars(v::DataArray{Any}) rmVars(convert(Array{Symbol},v)) end
    function rmVars(v::Array{Any}) rmVars(convert(Array{Symbol},v)) end
    function rmVars(v::Array{Symbol})
        return setdiff(vars,v)
    end
    vars=setdiff(names(dfd),   vcat(m[:exclude_vars],m[:y_var],:panid,cfg[:random_demos],cfg[:random_campaigns],cfg[:scoring_vars]) )
    upperModName = uppercase( string(  m[:modelName]  )  )
    println( upperModName*" : SingleValue") #SingleValue
    removed_SingleLevelVars=FS_singleLevel(dfd,vars)
    vars = rmVars(removed_SingleLevelVars)
    println(upperModName*" : Singularity : "*string(genF(m[:y_var],vars))) # Singularity
    singularity_x = checksingularity(genF(m[:y_var],vars), dfd)
    vars = rmVars(singularity_x)
    println(upperModName*" : PVals") #PVals
    f=genF(m[:y_var],vars)
    g1 = glm( f, dfd[convert(Array{Symbol},vcat(f.lhs,f.rhs.args[3:end]))]  , m[:dist], m[:lnk] )
    sdf = coefDF(g1,true)
    #g1_x= sdf[sdf[:pval].>0.7,:vars]
    g1_x= sdf[sdf[:pval].>0.7,:parameter]
    #vars = rmVars(g1_x)
    vars = rmVars(convert(Array{Any},g1_x))
    #function chkSigns(m::Dict, vars::Array{Symbol}, dfd::DataFrame, cfg::OrderedDict)  # Pvalue & Signs
    #    vars=unique(vcat(vars,[:group]))
    #    f=genF(m[:y_var],vars)
    #    g = glm( f, dfd[convert(Array{Symbol},vcat(f.lhs,f.rhs.args[3:end]))]  , m[:dist], m[:lnk] )
    #    sdf = coefDF(g1)
    #    neutralvars = setdiff(vars,vcat(cfg[:negativevars],cfg[:positivevars]))
    #    neg=intersect(cfg[:negativevars],sdf[sdf[:coef].<0,:vars])
    #    pos=intersect(cfg[:positivevars],sdf[sdf[:coef].>0,:vars])
    #    varstokeep = intersect(vcat(neutralvars, pos,neg) ,  sdf[sdf[:pval].<cfg[:pvalue_lvl] ,:vars] )
    #    varstokeep =  convert(Array{Symbol},varstokeep)
    #    return g, varstokeep
    #end
    function chkSigns(m::Dict, vars::Array{Symbol}, dfd::DataFrame, cfg::OrderedDict)  # Pvalue & Signs
        vars=unique(vcat(vars,[:group]))
        f=genF(m[:y_var],vars)
        g = glm( f, dfd[convert(Array{Symbol},vcat(f.lhs,f.rhs.args[3:end]))]  , m[:dist], m[:lnk] )
        sdf = coefDF(g1, true)
        neutralvars = setdiff(vars,vcat(cfg[:negativevars],cfg[:positivevars]))
        neg=intersect(cfg[:negativevars],sdf[sdf[:coef].<0,:parameter])
        pos=intersect(cfg[:positivevars],sdf[sdf[:coef].>0,:parameter])
        varstokeep = intersect(vcat(neutralvars, pos,neg) ,  sdf[sdf[:pval].<cfg[:pvalue_lvl] ,:parameter] )
        varstokeep =  convert(Array{Symbol},varstokeep)
        return g, varstokeep
    end

    println(upperModName*" : SIGN Check 1")
    (g3, initialvars) = chkSigns(m, vars, dfd, cfg)
    println(upperModName*" : SIGN Check 2")
    (g4, vars_2) = chkSigns(m, initialvars, dfd, cfg)
    function getCorrVars(m::RegressionModel, dfd::DataFrame, vars_2::Array{Symbol})
        sdf = coefDF(m, true)
        vars_2 = convert(Array{Symbol}, setdiff(vars_2, filter(x-> typeof(dfd[x]) in [PooledDataArray{String,UInt8,1}], vars_2)) )
        rm_lst=Symbol[]
        if (length(vars_2) > 1) & (   length(getColswithType("num", dfd, vars_2 ) ) > 1  )
            stackdf = corDFD(dfd,vars_2)
            stackdf[:variable_pval] = [ sdf[sdf[:parameter].==c,:pval][1]   for c in stackdf[:variable]]
            stackdf[:vars_pval] = [ sdf[sdf[:parameter].==c,:pval][1]   for c in stackdf[:vars]]
            stackdf[:most_Sig] = map((x,y) -> x < y ? "variable" : "vars" ,stackdf[:variable_pval],stackdf[:vars_pval])

            for row in eachrow(stackdf[(stackdf[:value].> 0.8) | (stackdf[:value].<-0.8),:])
                if row[:vars] == "group"
                    push!(rm_lst,row[:variable])
                elseif row[:variable] == "group"
                    push!(rm_lst,row[:vars])
                else
                    row[:most_Sig] == "variable" ? push!(rm_lst,row[:vars]) : push!(rm_lst,row[:variable])
                end
            end
        end
        return rm_lst
    end
    println(upperModName*" : Correlation")
    corrvars_x = getCorrVars(g4, dfd,vars_2  )
    vars_2 = setdiff(vars_2,corrvars_x)
    (g5, finalvars) =  chkSigns(m, convert(Array{Symbol},vars_2), dfd, cfg)

    #VIF is only valid on multiple cols
    if length(finalvars) > 2
        println( upperModName *" : Z & Vif") #Z & Vif
        f=genF(m[:y_var],finalvars)
        g2 = glm( f, dfd[convert(Array{Symbol},vcat(f.lhs,f.rhs.args[3:end]))]  , m[:dist], m[:lnk] )
        sdf=vifDF2(g2)
        z = sdf[abs(sdf[:zval]).<1.96,:parameter]
        v = sdf[ !DataArrays.isna(sdf[:vif])&(sdf[:vif].>15),:parameter]
        g2_x =intersect(z,v)
        finalvars = setdiff(finalvars,g2_x)
        println("vif_vars: ",g2_x)
    end
    finalvars = setdiff(finalvars,[:group])# reorder for group
    finalvars = convert(Array{Symbol},vcat(finalvars,[:group]) )
    return finalvars
end

factor_cols=vcat( [ cfg[:proscore], :group, :panid], cfg[:allrandoms] )
for c in setdiff(factor_cols,[:panid, cfg[:proscore]]) #cfg[:random_campaigns]
    if !( typeof(dfd[c]) in [  DataArray{String,1}  ])
        println("converting to Strings : ", c," of type : ",typeof(dfd[c]))
        dfd[c] = map(x->string(x),dfd[c])
        dfd[c] = convert(Array{String},dfd[c])
    end
end
poolit!(dfd,factor_cols)

iocc[:finalvars] = featureSelection(dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:], expandM(iocc))
idolocc[:finalvars]   = featureSelection(dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:], expandM(idolocc))
ipen[:finalvars]  = featureSelection(dfd[(dfd[:iso].==false) ,:], expandM(ipen))

"""
dfcor(dfr::DataFrame) = dfcor(dfr, names(dfr))

function dfcor(dfr::DataFrame, cols::Vector{Symbol})
    nms = Symbol[]
    arr = Float64[]
    for (n, v) in eachcol(dfr)
        if n ∈ cols && isnumeric(v)
            push!(nms, n)
            append!(arr, v)
        end
    end
    result = NamedArray(cor(reshape(arr, (size(dfr, 1), length(nms)))))
    NamedArrays.setnames!(result, string.(nms), 1)
    NamedArrays.setnames!(result, string.(nms), 2)
    result
end

dfcor1(dfr::DataFrame) = dfcor1(dfr, names(dfr))

function dfcor1(dfr::DataFrame, cols::Vector{Symbol})
    m = size(dfr, 1)
    nms = Symbol[]
    arr = ones(m)
    for (n, v) in eachcol(dfr)
        if n ∈ cols && isnumeric(v)
            push!(nms, n)
            append!(arr, v)
        end
    end
    n = length(nms) + 1
    R = full(UpperTriangular(view(qrfact!(reshape(arr, (m, n)))[:R],
                2:n, 2:n)))
    for j in 1:size(R, 2)
        colj = view(R, 1:j, j)
        colj ./= norm(colj)
    end
    Rt = UpperTriangular(R)
    result = NamedArray(Rt'Rt)
    NamedArrays.setnames!(result, string.(nms), 1)
    NamedArrays.setnames!(result, string.(nms), 2)
    result
end
"""
#m=expandM(ipen)
#glm(genF(m[:y_var],m[:finalvars]), dfd[vcat(m[:y_var],m[:finalvars])], m[:dist], m[:lnk])

# --- Removing outlliers - to ensure overall campaign uplift
function getOutliers(r::DataFrame, dfd::DataFrame, m::Dict, segment::Symbol,pct::Int64) # :data_nb_ne_b, :data_b_e_nb, :pen_reduction, :occ_reduction, :dolocc_reduction
    if m[:Buyer_Pos_P1_is1]
        println("pre=1")
        odf = dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1)&(dfd[segment].==true),:panid]
    else
        println("pre!=1")
        odf = dfd[(dfd[:iso].==false)&(dfd[segment].==true),:panid]
    end
    pctnum = Integer(round(length(odf) / 100) * pct)
    return sort(r[findin(r[:panid],odf) ,:], cols=[order(:resids, rev = true)] )[1:pctnum,:panid]
end

function genGLM(dfd::DataFrame, m::Dict)
    f=genF(m[:y_var],m[:finalvars])
    println("GLM for ",m[:modelName]," ::: ",f)
    dfd_tmp = m[:modelName] in [:occ, :dolocc] ? dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:] : dfd[(dfd[:iso].==false),:]
    g = glm(f, dfd_tmp[vcat(m[:y_var],m[:finalvars])], m[:dist], m[:lnk])
    println("GLM Residuals for ",m[:modelName])
    r = DataFrame(panid=dfd_tmp[:panid], resids=StatLib.xResiduals(g))
    return r, g
end

function genlists(dfd::DataFrame, iocc::Dict,idolocc::Dict,ipen::Dict)
    r1, g1 = genGLM( dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:], expandM(iocc))
    r2, g2 = genGLM( dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:], expandM(idolocc))
    r3, g3 = genGLM( dfd[(dfd[:iso].==false),:], expandM(ipen))
    rlist = [r1,r2,r3]
    glist = [g1,g2,g3]
    return rlist, glist
end
rlist, glist = genlists(dfd,iocc,idolocc,ipen)

for i in 1:5
    mlist = [expandM(iocc),expandM(idolocc),expandM(ipen)]
    #mliftβ = [ g.sdf[g.sdf[:vars].==:group,:coef][1] for g in glist]
    mliftβ = [ sdf[sdf[:parameter].==:group,:coef][1] for sdf in [coefDF(m, true) for m in glist]]
    minidx = find(x->x==minimum(mliftβ),mliftβ)[1]
    if  mliftβ[minidx] < 0.0
        #mx=mlist[minidx]
        println("WRONG : ",mliftβ," for ",mlist[minidx][:modelName])
        panids = getOutliers(rlist[minidx], dfd, mlist[minidx], :data_nb_ne_b,5)
        dfd[findin(dfd[:panid],panids) ,:iso] = true
        rlist, glist = genlists(dfd,iocc,idolocc,ipen)
    else
        println("All good : ",mliftβ)
        break
    end
end

# ---- SAVE RESULTS -----------
delete!(iocc, :exclude_vars)
delete!(idolocc, :exclude_vars)
delete!(ipen, :exclude_vars)

modelsDict = Dict()
modelsDict[:iocc] = iocc
modelsDict[:idolocc] = idolocc
modelsDict[:ipen] = ipen

modelsDict[:factors] = unpool(dfd)

cols = vcat([:panid, :iso, :group, :buyer_pos_p1],cfg[:random_campaigns])
cols = vcat(cols,iocc[:finalvars],[iocc[:y_var], iocc[:logvar]])
cols = vcat(cols,idolocc[:finalvars],[idolocc[:y_var], idolocc[:logvar]])
cols = unique(vcat(cols,ipen[:finalvars],[ipen[:y_var], ipen[:logvar]]) )
# add required cold for Scoring agregation
cols = unique(vcat(cols,[:buyer_pre_p1, :buyer_pos_p1, :trps_pos_p1,:trps_pre_p1,:dol_per_trip_pre_p1,:dol_per_trip_pos_p1, :prd_1_net_pr_pre, :prd_1_net_pr_pos]))

save_dfd(root, dfd[(dfd[:iso].==false),cols] )
save_modelsDict(root, modelsDict)


using Avalon

include("model.jl")
include("imagefolder.jl")

# this should point to a real folder containing subfolders with images
# each subfolder name will be taken as class name, while contained images
# will be used for training
# images can have any size, but will be resized to (100, 100) during loading
const IMAGE_DATASET_PATH = "~/data/food-101-sample"
# number of subfolders with images
const NUM_CLASSES = 5

# TODO:
# 4. load pretrained net
# 5. make wrappers like resnet18(), etc.

################################################################################
#                         TRAINING FROM SCRATCH                                #
################################################################################

# note that this method doesn't do validation or testing, but is only useful to make sure
# value of loss function is reduced over time
function Avalon.fit!(m::ResNet, dataset::ImageFolder, loss_fn;
              n_epochs=10, batch_size=100, opt=Adam(;lr=1e-3), device=CPU(), report_every=1)
    f = (m, x, y) -> loss_fn(m(x), y)
    num_batches = length(dataset) // batch_size
    for epoch in 1:n_epochs
        epoch_loss = 0
        for (i, (x, y)) in enumerate(Avalon.batchiter(dataset, sz=batch_size))
            x = device(copy(x))
            y = device(copy(y))
            loss, g = grad(f, m, x, y)
            update!(opt, m, g[2])
            epoch_loss += loss
            println("  batch loss = $loss")
        end
        if epoch % report_every == 0
            println("Epoch $epoch: avg_loss=$(epoch_loss / num_batches)")
        end
    end
    return m
end

function train_from_scretch()
    device = best_available_device()
    dataset = ImageFolder(expanduser(IMAGE_DATASET_PATH), transform=img -> imresize(img, (100, 100)))
    block = Bottleneck
    block_nums = [2, 2, 2, 2]
    stride = 1
    m = ResNet(block, block_nums, zero_init_residual=true, num_classes=NUM_CLASSES) |> device
    loss_fn = CrossEntropyLoss()
    fit!(m, dataset, loss_fn; device=device, batch_size=5)
end

################################################################################
#                         USING PRETRAINED MODEL                               #
################################################################################

# Will be added once ONNX import/export is implemented


using BERT
using Knet
import Base: length, iterate
using Random
using CSV
using PyCall

VOCABFILE = "bert-base-uncased-vocab.txt"
NUM_CLASSES = 2

token2int = Dict()
f = open(VOCABFILE) do file
    lines = readlines(file)
    for (i,line) in enumerate(lines)
        token2int[line] = i
    end
end
int2token = Dict(value => key for (key, value) in token2int)
VOCABSIZE = length(token2int)

mutable struct ClassificationData2
    input_ids
    input_mask
    segment_ids
    labels
    batchsize
    ninstances
    shuffled
end

function ClassificationData2(input_file; batchsize=8, shuffled=true, seq_len=64)
    input_ids = []
    input_mask = []
    segment_ids = []
    labels = []
    f = open(input_file)
    tmp = split.(readlines(f), "\t")
    for i in 1:length(tmp)
        instance = eval.(Meta.parse.(tmp[i]))
        push!(input_ids, (instance[1] .+ 1)[1:seq_len])
        push!(input_mask, instance[2][1:seq_len])
        push!(segment_ids, (instance[3] .+ 1)[1:seq_len])
        push!(labels, (instance[4] + 1))
    end
    ninstances = length(input_ids)
    return ClassificationData2(input_ids, input_mask, segment_ids, labels, batchsize, ninstances, shuffled)
end

function length(d::ClassificationData2)
    d, r = divrem(d.ninstances, d.batchsize)
    return r == 0 ? d : d+1
end

function iterate(d::ClassificationData2, state=ifelse(d.shuffled, randperm(d.ninstances), 1:d.ninstances))

    state === nothing && return nothing

    if length(state) > d.batchsize
        new_state = state[d.batchsize+1:end]
        input_ids = hcat(d.input_ids[state[1:d.batchsize]]...)
        input_mask = hcat(d.input_mask[state[1:d.batchsize]]...)
        segment_ids = hcat(d.segment_ids[state[1:d.batchsize]]...)
        labels = hcat(d.labels[state[1:d.batchsize]]...)
    else
        new_state = nothing
        input_ids = hcat(d.input_ids[state]...)
        input_mask = hcat(d.input_mask[state]...)
        segment_ids = hcat(d.segment_ids[state]...)
        labels = hcat(d.labels[state]...)
    end

    return ((input_ids, input_mask, segment_ids, labels), new_state)
end

# Embedding Size, Vocab Size, Intermediate Hidden Size, Max Sequence Length, Sequence Length, Num of Segments, Num of Heads in Attention, Num of Encoders in Stack, Batch Size, Matrix Type, General Dropout Rate, Attention Dropout Rate, Activation Function
config = BertConfig(768, 30522, 3072, 512, 64, 2, 12, 12, 8, KnetArray{Float32}, 0.1, 0.1, "gelu")

dtst = ClassificationData2("../project/sst-test.tsv", batchsize=config.batchsize, seq_len=config.seq_len)

model = BertClassification(config, NUM_CLASSES)

@pyimport torch
torch_model = torch.load("../project/model-64-32.pt")

model = load_from_torch_classification(model, config.num_encoder, config.atype, torch_model)

function accuracy2(model, dtst)
    true_count = 0
    all_count = 0
    for (x, attention_mask, segment_ids, y) in dtst
        probs = model(x, segment_ids, attention_mask=attention_mask)
        preds = map(x -> x[1], argmax(Array{Float32}(probs),dims=1))
        true_count += sum(y .== preds)
        all_count += length(y)
    end
    return true_count/all_count
end

result = accuracy2(model, dtst)

println("Test accuracy is : $result")


module CG

using IterativeSolvers, LinearAlgebra

# define interface to call a function pointer as a matrix-free operator
struct COp
    fptr::Ptr{Cvoid}
    n::Int
end

Base.eltype(A::COp) = Float64
Base.size(A::COp, d) = d <= 2 ? A.n : 1

function LinearAlgebra.mul!(y::StridedVector{Float64}, A::COp, x::StridedVector{Float64})
    @assert stride(x,1) == 1
    @assert stride(y,1) == 1
    i = ccall(A.fptr, Cint, (Ptr{Cdouble}, Ptr{Cdouble}), y, x)
    i == 0 || throw("Oh no")
    return y
end
function Base.:*(A::COp, x::AbstractVector{Float64})
    LinearAlgebra.mul!(similar(x), A, x)
end

Base.@ccallable function julia_cg(fptr::Ptr{Cvoid}, cx::Ptr{Cdouble}, cb::Ptr{Cdouble}, len::Csize_t)::Cint
    try
        x = unsafe_wrap(Array, cx, (len,))
        b = unsafe_wrap(Array, cb, (len,))
        A = COp(fptr,len)
        cg!(x, A, b)
    catch
        Base.invokelatest(Base.display_error, Base.catch_stack())
        return 1
    end
    return 0
end

end # module


function search!(actr, ex)
    status = :searching
    # search until target found or termination threshold met
    while status == :searching
        # update decay values of objects in iconic memory
        update_decay!(actr)
        # update the finst values for return of inhabition
        update_finst!(actr)
        # update which items are in iconic memory based on what is currently visible
        update_visibility!(actr)
        # update the activations
        compute_activations!(actr)
        ex.visible ? update_window!(actr, ex) : nothing
        # perform search sequence
        status = _search!(actr, ex)
        ex.visible ? update_window!(actr, ex) : nothing
    end
    # add data for trial
    add_data!(ex)
    # add fixations for trial
    push!(ex.fixations, ex.trial_fixations)
    return nothing
end

function _search!(actr, ex)
    ex.trace ? print_trial(ex) : nothing
    data = ex.trial_data
    # production cycle
    cycle_time!(actr, ex)
    # select visual object with highest activation
    status = find_object!(actr, ex)
    if status == :error
        # if not objects are found, add respond time
        cycle_time!(actr, ex)
        motor_time!(actr, ex)
        # add fixation data
        add_no_fixation!(ex, actr)
        ex.trace ? print_response(actr, "absent") : nothing
        # add response to data
        add_response!(actr, data, :absent)
        return status
    end
    # Attending object in abstract-location
    # add cycle time
    cycle_time!(actr, ex)
    # add attend time
    attend_time!(actr, ex)
    # place object in visual buffer and update attend time
    attend_object!(actr, ex)
    # add fixation to fixation data
    add_fixation!(ex, actr)
    # cycle time before response
    cycle_time!(actr, ex)
    # check if current object matches target
    status = check_object(actr, ex)
    if status == :present
        # if matches, respond present and collect data
        cycle_time!(actr, ex)
        motor_time!(actr, ex)
        ex.trace ? print_response(actr, "present") : nothing
        add_response!(actr, data, status)
        return status
    end
    # if does not match, return "searching" and continue
    return status
end

function motor_time!(actr, ex)
    θ = gamma_parms(.065)
    tΔ = rand(Gamma(θ...))
    actr.time += tΔ
    ex.visible ? sleep(tΔ/ex.speed) : nothing
    return nothing
 end

function cycle_time!(actr, ex)
    tΔ = .05
    if actr.parms.rnd_time
        θ = gamma_parms(.05)
        tΔ = rand(Gamma(θ...))
    end
    actr.time += tΔ
    ex.visible ? sleep(tΔ/ex.speed) : nothing
    return nothing
end

function attend_time!(actr, ex)
    # tΔ = saccade_time(model) + visual_encoding(model)
    tΔ = visual_encoding(actr)
    actr.time += tΔ
    ex.visible ? sleep(tΔ/ex.speed) : nothing
    return nothing
end

saccade_time(actr) = saccade_time(actr, actr.visual_location.buffer[1])

function saccade_time(actr, vo)
    @unpack Δexe, β₀exe = actr.parms
    distance = compute_angular_distance(actr, vo)
    μ = β₀exe + distance*Δexe
    θ = gamma_parms(μ)
    return rand(Gamma(θ...))
end

function visual_encoding(actr)
    if actr.parms.rnd_time
        θ = gamma_parms(.085)
        return rand(Gamma(θ...))
    end
    return .085
end

gamma_parms(μ, σ) = (μ/σ)^2,σ^2/μ

gamma_parms(μ) = gamma_parms(μ, μ/3)

function add_fixation!(ex, actr)
    goal = get_buffer(actr, :goal)
    vision = get_buffer(actr, :visual)
    visicon = actr.visicon
    attend_time = vision[1].attend_time
    slots = goal[1].slots
    idx = findfirst(x->x.location == vision[1].location, visicon)
    fixation = Fixation(; attend_time, color=slots.color,
        shape=slots.shape, idx, stop=false)
    push!(ex.trial_fixations, fixation)
    return nothing
end

function add_no_fixation!(ex, actr)
    goal = get_buffer(actr, :goal)
    vision = get_buffer(actr, :visual)
    visicon = actr.visicon
    attend_time = actr.time
    slots = goal[1].slots
    idx = length(visicon) + 1
    fixation = Fixation(; attend_time, color=slots.color,
        shape=slots.shape, idx, stop=true)
    push!(ex.trial_fixations, fixation)
    return nothing
end

function add_data!(ex)
    push!(ex.data, ex.trial_data)
    return nothing
end

function add_response!(actr, data, status)
    data.response = status
    data.rt = actr.time
    set_trial_type!(data)
    return nothing
end

function relevant_object(actr, vo)
    !vo.visible || vo.attended ? (return false) : (return true)
end

function find_object!(actr, ex)
    v = :_
    if actr.parms.noise
        v = find_object2!(actr, ex)
    else
        v = find_object1!(actr, ex)
    end
    return v
end

function find_object1!(actr, ex)
    @unpack iconic_memory = actr.visual_location
    visible_objects = filter(x->relevant_object(actr, x), iconic_memory)
    if isempty(visible_objects)
        ex.trace ? print_abstract_location(actr, "error locating object") : nothing
        return :error
    end
    p = fixation_probs(actr, actr.visicon, visible_objects)
    idx = sample(1:length(p), Weights(p))
    if idx == length(p)
        ex.trace ? print_abstract_location(actr, "termination threshold exceeded") : nothing
        return :error
    end
    max_vo = visible_objects[idx]
    actr.visual_location.buffer = [max_vo]
    ex.trace ? print_abstract_location(actr, "object found") : nothing
    return :searching
end

function find_object2!(actr, ex)
    visible_objects = filter(x->relevant_object(actr, x), get_iconic_memory(actr))
    if isempty(visible_objects)
        ex.trace ? print_abstract_location(actr, "error locating object") : nothing
        return :error
    end
    max_vo = max_activation(visible_objects)
    if terminate(actr, max_vo)
        ex.trace ? print_abstract_location(actr, "termination threshold exceeded") : nothing
        return :error
    end
    actr.visual_location.buffer = max_vo
    ex.trace ? print_abstract_location(actr, "object found") : nothing
    return :searching
end

attend_object!(actr, ex) = attend_object!(actr, ex, actr.visual_location.buffer[1])

function attend_object!(actr, ex, vo)
    ex.trace ? print_vision(actr) : nothing
    actr.visual.buffer = [vo]
    vo.attended = true
    vo.attend_time = actr.time
    actr.visual.focus = vo.location
    ex.trace ? print_visual_buffer(actr) : nothing
    return nothing
end

check_object(actr, ex) = check_object(actr, actr.visual.buffer[1], ex, actr.goal.buffer[1])

function check_object(actr, vo, ex, target)
    for (f,v) in pairs(target.slots)
        if vo.features[f].value ≠ v
            ex.trace ? print_check(actr, "not found") : nothing
            update_threshold!(actr)
            return :searching
        end
    end
    ex.trace ? print_check(actr, "found") : nothing
    return :present
end

function max_activation(vos)
    max_vo = similar(vos, 1)
    mv = -Inf
    for vo in vos
        if vo.activation > mv
            max_vo[1] = vo
            mv = vo.activation
        end
    end
    return max_vo
end

function update_threshold!(actr)
    actr.parms.τₐ += actr.parms.Δτ
end

terminate(actr, max_vo) = terminate(actr, max_vo[1])

function terminate(actr, max_vo::VisualObject)
    τ = actr.parms.τₐ + add_noise(actr)
    τ > max_vo.activation ? (return true) : (return false)
end

function update_decay!(actr)
    map(x->update_decay!(actr, x), get_iconic_memory(actr))
end

function update_decay!(actr, object)
    for f in object.features
        if actr.parms.persistence < (actr.time - f.fixation_time)
            f.visible = false
            f.fixation_time = 0.0
        end
    end
    return nothing
end

function update_finst!(actr)
    attended_objects = filter(x->x.attended, get_iconic_memory(actr))
    isempty(attended_objects) ? (return) : nothing
    map(x->update_finst_span!(actr, x), attended_objects)
    update_n_finst!(actr, attended_objects)
end

function update_finst_span!(actr, vo)
    if actr.parms.finst_span < (actr.time - vo.attend_time)
        vo.attended = false
        vo.attend_time = 0.0
    end
    return nothing
end

function update_n_finst!(actr, vos)
    N = length(vos) - actr.parms.n_finst
    N <= 0 ? (return) : nothing
    sort!(vos, by=x->x.attend_time)
    for i in 1:N
        vos[i].attended = false
        vos[i].attend_time = 0.0
    end
    return nothing
end

function compute_activations!(actr)
    @unpack iconic_memory = actr.visual_location
    iconic_memory = filter(x->x.visible, actr.visicon)
    topdown_activations!(iconic_memory, actr.goal.buffer[1])
    bottomup_activations!(iconic_memory)
    weighted_activations!(actr, iconic_memory)
    actr.parms.noise ? add_noise!(actr, iconic_memory) : nothing
    return nothing
end

function weighted_activations!(model, iconic_memory)
    for object in iconic_memory
        weighted_activation!(model, object)
    end
    return nothing
end

function weighted_activation!(actr, vo)
    vo.activation = weighted_activation(actr, vo)
    return nothing
end

function weighted_activation(actr, vo)
    return actr.parms.bottomup_weight * vo.bottomup_activation +
    actr.parms.topdown_weight * vo.topdown_activation
end

bottomup_activations!(actr::ACTRV) = bottomup_activations!(get_iconic_memory(actr))

function bottomup_activations!(iconic_memory)
    for vo1 in iconic_memory
        activation = 0.0
        for vo2 in iconic_memory
            vo1 == vo2 ? continue : nothing
            activation += bottomup_activation(vo1, vo2)
        end
        vo1.bottomup_activation = activation
    end
    return nothing
end

function bottomup_activation(vo1, vo2)
    distance = compute_distance(vo1, vo2)
    return bottomup_activation(vo1.features, vo2.features, distance)
end

function bottomup_activation(f1, f2, distance)
    activation = 0.0
    for (f,v) in pairs(f1)
        if (f2[f].value ≠ v.value) && v.visible && f2[f].visible
            activation += 1.0/sqrt(distance)
        end
    end
    return activation
end

topdown_activations!(iconic_memory, chunk::Chunk) = topdown_activations!(iconic_memory, chunk.slots)

function topdown_activations!(iconic_memory, target)
    for object in iconic_memory
        topdown_activation!(object, target)
    end
    return nothing
end

function topdown_activation!(object, target)
    activation = 0.0
    for (f,v) in pairs(object.features)
        if v.visible
            if v.value == target[f]
                activation += 1.0
            end
        else
            activation += .5
        end
    end
    object.topdown_activation = activation
    return nothing
end

add_noise(actr) = rand(Normal(0, actr.parms.σ))

add_noise!(actr, vo::VisualObject) = vo.activation += add_noise(actr)

add_noise!(actr, iconic_memory) = map(x->add_noise!(actr, x), iconic_memory)

function update_visibility!(actr)
    map(x->feature_visibility!(actr, x), get_iconic_memory(actr))
    map(x->object_visibility!(x), get_iconic_memory(actr))
end

function feature_visibility!(actr, vo)
    angular_distance = compute_angular_distance(actr, vo)
    for (f,v) in pairs(vo.features)
        parms = actr.parms.acuity[f]
        threshold = compute_acuity_threshold(parms, angular_distance)
        if feature_is_visible(vo, threshold)
            v.visible = true
        end
    end
    return nothing
end

feature_is_visible(vo::VisualObject, threshold) = feature_is_visible(vo.angular_size, threshold)

function feature_is_visible(angular_size, threshold)
    return angular_size > threshold
end

function object_visibility!(object)
    for f in object.features
        if f.visible
            object.visible = true
            return nothing
        end
    end
    object.visible = false
    return nothing
end

function compute_distance(actr::ACTRV, object)
    sqrt(sum((actr.visual.focus .- object.location).^2))
end

function compute_distance(vo1, vo2)
    sqrt(sum((vo1.location .- vo2.location).^2))
end

"""
Angular distance in degress. Also known as eccentricity
"""
function compute_angular_distance(actr, vo)
    distance = compute_distance(actr, vo)
    return pixels_to_degrees(actr, distance)
end

function compute_angular_size!(actr)
    map(x->compute_angular_size!(actr, x), get_visicon(actr))
    return nothing
end

function compute_angular_size!(actr, vo)
    ppi = 72 # pixels per inch
    distance = actr.parms.viewing_distance * ppi
    vo.angular_size = compute_angular_size(distance, vo.width)
end

function compute_angular_size(distance, width)
    radians = 2 * atan(width / (2 * distance))
    return rad2deg(radians)
end

rad2deg(radians) = radians * 180 / pi

function compute_acuity_threshold(parms, angular_distance)
    return parms.a * angular_distance^2 - parms.b * angular_distance
end

function pixels_to_degrees(actr, pixels)
    ppi = 72 # pixels per inch
    radians = atan(pixels/ppi, actr.parms.viewing_distance)
    return rad2deg(radians)
end

function orient!(actr, ex)
    w = ex.array_width / 2
    actr.visual.focus = fill(w, 2)
end

function fixation_prob(actr, visicon, visible_objects, fixation)
    @unpack stop,idx = fixation
    act = map(x->x.activation, visible_objects)
    push!(act, actr.parms.τₐ)
    σ = actr.parms.σ*sqrt(2)
    vo_act = stop ? act[end] : visicon[idx].activation
    return exp(vo_act/σ)/sum(exp.(act/σ))
end

function fixation_probs(actr, visicon, visible_objects)
    act = map(x->x.activation, visible_objects)
    push!(act, actr.parms.τₐ)
    σ = actr.parms.σ * sqrt(2)
    return exp.(act / σ) / sum(exp.(act / σ))
end

# visual_encoding(model) = visual_encoding(model, model.abstract_location[1])
#
# function visual_encoding(model, vo)
#     frequency = model.init_freq
#     distance = compute_angular_distance(model, vo)
#     μ = model.K_encode*-log(frequency)*exp(distance*model.κ_encode)
#     θ = gamma_parms(μ)
#     # vo.encode_start = model.current_time
#     # vo.encode_time = rand(Gamma(a, b))
#     return rand(Gamma(θ...))
# end


using Images, StaticArrays, LinearAlgebra, Statistics
using FileIO, ProgressMeter, Printf, ImageSegmentation
#using Plots, ImageView

include("psf_fun.jl")
include("s_save_image.jl")
include("filter.jl")
include("psf_analyze.jl")
include("beadstest.jl")

THRESHOLD = 7

print("Disk label: ")
label = chomp(readline())

print("Date: ")
date = chomp(readline())
#date="20190315"
@show date

cd(@sprintf("/mnt/%s/jchang/%s/", label, date))
files = readdir()
filelist = filter(x -> occursin(".imagine", x), files)
@show filelist

print("add scale bar(Y as 1, N as 0): ")
r = chomp(readline())
r = parse(Int, r)

l = 0
if r == 1
    print("The length of scale bar(in px): ")
    l = chomp(readline())
    l = parse(Int, l)
end

print("Background Filename: ")
BG_filename = chomp(readline())

if BG_filename == ""
    BG_filename = filelist[1][1:end-8]
    BG_filename *= "_1"
end

print("Save every file during process (1 as Yes, 0 as No): ")
Save = chomp(readline())
Save = (Save == "1")

for k = 1:size(filelist, 1)
    filename = filelist[k][1:end-8]
    exp = load(@sprintf("%s.imagine", filename))

    transfertotif(exp, filename, r, l) # save .imagine into .tif
    files = readdir()
    framelist = filter(x -> occursin(".tif", x), files)
    framelist = filter(x -> occursin(@sprintf("%s", filename), x), framelist)

    for i = 1:size(framelist, 1)
        filename = framelist[i][1:end-4]
        exp = load(@sprintf("%s.tif", filename))

        #img = BG_subtraction(exp,BG_filename,filename,Save)# filter image with background signal
        img = convert(Array{N0f16}, exp)
        img = beads_segment(img, THRESHOLD, 10000, 3)
        #psf_analyze(img,filename)
        img3 = Gray.(convert(Array{N0f16}, OffsetArrays.no_offset_view(img)))
        img4 = Gray.(convert.(Normed{UInt16,16}, img3))
        img_save(img, "/home/jchang/image/result/", @sprintf("%s-fi.tif", filename))
    end
end


const NTypes{N} = NTuple{N, Val}

valueof(::Val{x}) where {x} = x

# Not exactly `Base.aligned_sizeof`
Base.@pure function sizeof_aligned(T::Type)
    if isbitstype(T)
        al = Base.datatype_alignment(T)
        return (Core.sizeof(T) + al - 1) & -al
    else
        return nothing
    end
end

@inline foldlargs(op, x) = x
@inline foldlargs(op, x1, x2, xs...) = foldlargs(op, @return_if_reduced(op(x1, x2)), xs...)

terminating_foldlargs(op, fallback) = fallback()
@inline function terminating_foldlargs(op, fallback::F, x1, x2, xs...) where {F}
    acc = op(x1, x2)
    acc isa Reduced && return unreduced(acc)
    xs isa Tuple{} && return fallback()
    return terminating_foldlargs(op, fallback, acc, xs...)
end

@inline foldrunion(op, ::Type{T}, init) where {T} =
    if T isa Union
        acc = @return_if_reduced foldrunion(op, T.b, init)
        foldrunion(op, T.a, acc)
    else
        op(T, init)
    end

@generated asntypes(::Type{T}) where {T} =
    QuoteNode(foldrunion((S, types) -> (Val(S), types...), T, ()))

struct UnionValue{T <: NTypes,NBytes}
    types::T
    data::NTuple{NBytes,UInt32}
    typeid::UInt8
end

@noinline unreachable() = error("unreachable")

@inline function unionvalue(::Type{T}, v::T) where {T}
    if T isa Union
        types = asntypes(T)
        nbytes = foldrunion(T, 0) do S, n
            Base.@_inline_meta
            max(sizeof_aligned(S), n)
        end
        dest = Ref(ntuple(_ -> UInt32(0), Val(nbytes)))
        GC.@preserve dest begin
            unsafe_store!(Ptr{typeof(v)}(pointer_from_objref(dest)), v)
        end
        @inline function searchid((v, id), t)
            if v isa valueof(t)
                Reduced(id)
            else
                (v, id + 1)
            end
        end
        typeid = terminating_foldlargs(searchid, unreachable, (v, 1), types...)
        return UnionValue(types, dest[], UInt8(typeid))
    else
        return v
    end
end

@noinline invalid_typeid() = error("invalid typeid")

interpret(x) = x
@inline function interpret(uv::UnionValue)
    data = uv.data
    typeid = uv.typeid
    @inline function _get(id, t)
        if id == typeid
            T = valueof(t)
            ref = Ref(data)
            GC.@preserve ref begin
                v = unsafe_load(Ptr{T}(pointer_from_objref(ref)))
            end
            return Reduced(v)
        else
            id + 1
        end
    end
    return terminating_foldlargs(_get, invalid_typeid, 1, uv.types...)
end

@inline function CUDA.shfl_recurse(op, uv::UnionValue)
    data = map(op, uv.data)
    typeid = op(uv.typeid)
    return UnionValue(uv.types, data, typeid)
end


function rnucleus(Z, mass_number, M,
                  spin,
                  dip_moment, quad_moment)
    pipe_file_run("$grasp/rnucleus",
                  """$Z ! Nuclear charge
$mass_number ! Nuclear mass number
n ! Don't revise dimensions
$M ! Mass of neutral atom
$spin ! Nuclear spin [ℏ]
$dip_moment ! Nuclear dipole moment [nuclear magnetons]
$quad_moment ! Nuclear quadruple moment [barn]
n ! Don't revise grid

""")
end

function rcsfgenerate_input(active,
                            configurations,
                            j::Range,
                            nexc)
    """$(join(configurations, '\n')) ! Configurations

$active
$(j[1]),$(j[end]) ! jmin,jmax
$nexc ! Number of excitations"""
end

rcsfgenerate_input(active,
                   configurations,
                   term::AbstractString,
                   nexc) = rcsfgenerate_input(active,
                                              configurations,
                                              term_to_2j_range(term),
                                              nexc)
function rcsfgenerate(core, active, lists)
    # If core not in list, the value is zero (no core)
    core = findfirst([:He :Ne :Ar :Kr :Xe :Rn], core)
    lists = join(map(l -> rcsfgenerate_input(active, l...), lists), "\ny\n")
    pipe_file_run("$grasp/rcsfgenerate",
                  """* ! Default ordering
$core ! Core
$lists
n ! No more lists

""")
    cpf("rcsfgenerate.log", "$(active_file(active)).exc")
    cpf("rcsf.out", "rcsf.inp")
end

function rangular()
    pipe_file_run("$grasp/rangular",
                  "y ! Default parameters\n")
end

function rwfnestimate()
    # For interactive runs
    run(`$grasp/rwfnestimate`)
end

function rwfnestimate(estimate...)
    pipe_file_run("$grasp/rwfnestimate",
                  """y ! Default parameters
$(join(map(e -> join(e, '\n'), estimate), "\n*\n"))
*
""")
end

function rmcdhf(vary, spectroscopic, cycles=100)
    no_blocks = length(split(readall("rcsf.inp"), "*"))
    pipe_file_run("$grasp/rmcdhf",
                  """y ! Default parameters
$(join(["1" for i = 1:no_blocks], '\n')) $(no_blocks > 1 ? "\n5" : "")
$vary
$spectroscopic
$cycles
""")
end

function rmcdhf()
    # For interactive runs
    run(`$grasp/rmcdhf`)
end

function grasp_save_run(name)
    run(`$grasp/rsave $(active_file(name))`)
end

function grasp_cp_wfn(a,b)
    mkpath("$b")
    cpf("$a/rwfn.out", "$b/rwfn.out")
end

function parse_eng(level, J, parity,
                   Hartrees,
                   Kaysers,
                   eV)
    Dict(:level => parse(Int, level),
         :J => parse(Int, split(J, "/")[1])//2,
         :parity => parse(Int, "$(parity)1"),
         :Hartrees => parse(Float64, replace(Hartrees, "D", "e")),
         :Kayers => parse(Float64, replace(Kaysers, "D", "e")),
         :eV => parse(Float64, replace(eV, "D", "e")))
end

function read_eng(name)
    open("$(active_file(name)).sum") do file
        for line in enumerate(eachline(file))
            strip(line[2]) == "Eigenenergies:" && break
        end
        readline(file)
        readline(file)
        readline(file)
        energies = []
        for line in eachline(file)
            strip(line) == "" && break
            push!(energies, parse_eng(split(strip(line))...))
        end
        energies
    end
end

function grasp_clean()
    # Clean up temporary files that take up a lot of space
    for f in filter(f -> contains(f, "mcp."), readdir())
        rm(f)
    end
end

export rnucleus, rcsfgenerate_input, rcsfgenerate, rangular, rwfnestimate, rmcdhf, grasp_save_run, grasp_cp_wfn, parse_eng, read_eng, grasp_clean


using AutomotiveSensors
using AutomotiveSimulator
using Test

@testset "occlusion checker" begin
    obs = ConvexPolygon([VecE2(0.0, 0.0), VecE2(10.0, 0.0), VecE2(10.0, 10.0), VecE2(0.0, 10.0)])
    veh1 = VehicleState(VecSE2(-5,-5,0.0), 0.)
    veh2 = VehicleState(VecSE2(15,15,0.0), 0.0)
    veh3 = VehicleState(VecSE2(10.0, -5, 0.0), 0.)

    @test occlusion_checker(veh1, veh2, obs)
    @test !occlusion_checker(veh1, veh3, obs)
end

@testset "perfect sensor" begin
    sensor = PerfectSensor()
    roadway = gen_straight_roadway(1, 100.0)

    ego = Entity(VehicleState(VecSE2(20.0, 0.0, 0.0), 5.0), VehicleDef(), 1)
    veh1 = Entity(VehicleState(VecSE2(30.0, 0.0, 0.0), 5.0), VehicleDef(), 2)
    veh2 = Entity(VehicleState(VecSE2(10.0, 0.0, 0.0), 5.0), VehicleDef(), 3)
    scene = Scene([ego, veh1, veh2])

    obs = measure(sensor, ego, scene, roadway, ConvexPolygon[])

    @test obs[1] == veh1
    @test obs[2] == veh2
end

@testset "gaussian sensor" begin
    sensor = GaussianSensor(false_positive_rate=0.0, false_negative_rate=0.0)
    roadway = gen_straight_roadway(1, 100.0)

    ego = Entity(VehicleState(VecSE2(20.0, 0.0, 0.0), 5.0), VehicleDef(), 1)
    veh1 = Entity(VehicleState(VecSE2(30.0, 0.0, 0.0), 5.0), VehicleDef(), 2)
    veh2 = Entity(VehicleState(VecSE2(10.0, 0.0, 0.0), 5.0), VehicleDef(), 3)
    scene = Scene([ego, veh1, veh2])

    obs = measure(sensor, ego, scene, roadway, ConvexPolygon[])

    @test length(obs) == 2

    @test AutomotiveSensors.obs_weight(sensor, ego.state, obs[1].state, veh1.state, ConvexPolygon[]) > 0.0
end


__precompile__()
module JAMSDWriter

import MathProgBase
const MPB = MathProgBase
#importall MathProgBase.SolverInterface

using Compat
using Compat.LinearAlgebra, Compat.SparseArrays

if VERSION >= v"0.7"
    # quickfix for Nullable
    using Nullables
end

export JAMSDSolver, getsolvername, getsolveresult, getsolveresultnum, getsolvemessage, getsolveexitcode, LinearQuadraticModel

# Load in `deps.jl`, complaining if it does not exist
const depsjl_path = joinpath(dirname(@__FILE__), "..", "deps", "deps.jl")
if !isfile(depsjl_path)
    error("JAMSDWriter not installed properly, run Pkg.build(\"JAMSDWriter\"), restart Julia and try again")
end

include(depsjl_path)

const CONFIG = Dict(
:debug => false,
:export_gms => false,
:solver_log => false
)

const solverdata_dir = joinpath(dirname(@__DIR__), ".solverdata")

include("jamsd_linearity.jl")
include("jamsd_params.jl")
include("jamsd_convert.jl")
include("jamsd_fun.jl")

const solver_stat = [
   :Optimal,
   :Iteration,
   :Resource,
   :Solver,
   :EvalError,
   :Capability,
   :License,
   :User,
   :SetupErr,
   :SolverErr,
   :InternalErr,
   :Skipped,
   :SystemErr
]

const model_stat = [
   :OptimalGlobal,
   :OptimalLocal,
   :Unbounded,
   :InfeasibleGlobal,
   :InfeasibleLocal,
   :InfeasibleIntermed,
   :Feasible,
   :Integer,
   :NonIntegerIntermed,
   :IntegerInfeasible,
   :LicenseError,
   :ErrorUnknown,
   :ErrorNoSolution,
   :NoSolutionReturned,
   :SolvedUnique,
   :Solved,
   :SolvedSingular,
   :UnboundedNoSolution,
   :InfeasibleNoSolution
]

mutable struct JAMSDSolver <: MPB.AbstractMathProgSolver
    solver_name::String
    options::Dict{String,Any}
    emp::Nullable{Function}
end

# TODO(xhub) write a better struct/enum here
@enum MODEL_TYPE qcp=7 nlp=2 miqcp=6 minlp=5 emp=9

"change the debug state"
setdebug(b::Bool) = global CONFIG[:debug] = b
"export the problem to a GAMS Model file (.gms)"
setexport(b::Bool) = global CONFIG[:export_gms] = b
"printout the log from the solver"
setsolverlog(b::Bool) = global CONFIG[:solver_log] = b

"""
Create a JAMSDSolver Solver for MPB. The optional arguments are:

# Optional Arguments
- `solver_name::String=""`: solver used for this problem
- `options::Dict{String,Any}=Dict{String,Any}()`: the JAMSD options

"""
function JAMSDSolver(solver_name::String="",
                     options::Dict{String,Any}=Dict{String,Any}())
    JAMSDSolver(solver_name, options, Nullable{Function}())
end

getsolvername(s::JAMSDSolver) = basename(s.solver_name)

mutable struct JAMSDMathProgModel <: MPB.AbstractMathProgModel
    options::Dict{String, Any}

    solver_name::String

    x_l::Vector{Float64}
    x_u::Vector{Float64}
    g_l::Vector{Float64}
    g_u::Vector{Float64}

    nvar::Int
    ncon::Int

    obj
    constrs::Vector{Any}

    lin_constrs::Vector{Dict{Int, Float64}}
    lin_obj::Dict{Int, Float64}

    r_codes::Vector{Int}
    j_counts::Vector{Int}

    vartypes::Vector{Symbol}
    varlinearities_con::Vector{Symbol}
    varlinearities_obj::Vector{Symbol}
    conlinearities::Vector{Symbol}
    objlinearity::Symbol

    v_index_map::Dict{Int, Int}
    v_index_map_rev::Dict{Int, Int}

    # was:
    # c_index_map::Dict{Int, Int}
    # c_index_map_rev::Dict{Int, Int}

    nonquad_idx::Dict{Int, Int}
    quad_idx::Dict{Int, Int}

    sense::Symbol

    x_0::Vector{Float64}

    objval::Float64
    solution::Vector{Float64}

    status::Symbol
    solve_exitcode::Int
    solve_result_num::Int
    solve_result::String
    model_result_num::Int
    model_result::String
    solve_message::String
    solve_time::Float64

    model_type::MODEL_TYPE

    quad_equs::Vector{Any}
#    quad_obj::Tuple{Vector{Int}, Vector{Int}, Vector{Float64}}
    quad_obj::Tuple
    offset::Int
    emp::Nullable{Function}

    d::Nullable{MPB.AbstractNLPEvaluator}

    jamsd_ctx::Ptr{context}
    jamsd_ctx_dest::Ptr{context}
    jamsd_options::Ptr{jamsd_options}
    gams_dir::String

    function JAMSDMathProgModel(solver_name::String,
                                options::Dict{String,Any},
                                model_type::MODEL_TYPE,
                                emp)
        o = new(options,
            solver_name,
            zeros(0),
            zeros(0),
            zeros(0),
            zeros(0),
            0,
            0,
            :(0),
            [],
            Dict{Int, Float64}[],
            Dict{Int, Float64}(),
            Int[],
            Int[],
            Symbol[],
            Symbol[],
            Symbol[],
            Symbol[],
            :Lin,
            Dict{Int, Int}(),
            Dict{Int, Int}(),
            Dict{Int, Int}(),
            Dict{Int, Int}(),
            :Min,
            zeros(0),
            NaN,
            zeros(0),
            :NotSolved,
            -1,
            -1,
            "?",
            -1,
            "?",
            "",
            NaN,
            model_type,
            Vector{Any}(),
            (),
            0,
            emp,
            Nullable{MPB.AbstractNLPEvaluator}(),
            Ptr{context}(C_NULL),
            Ptr{context}(C_NULL),
            Ptr{jamsd_options}(C_NULL),
            "")
        @compat finalizer(jamsd_cleanup, o)
        o
    end
end

struct JAMSDLinearQuadraticModel <: MPB.AbstractLinearQuadraticModel
    inner::JAMSDMathProgModel
end

struct JAMSDNonlinearModel <: MPB.AbstractNonlinearModel
    inner::JAMSDMathProgModel
end

struct JAMSDConicModel <: MPB.AbstractNonlinearModel
    inner::JAMSDMathProgModel
end

include("jamsd_write.jl")

MPB.NonlinearModel(s::JAMSDSolver) = JAMSDNonlinearModel(
    JAMSDMathProgModel(s.solver_name, s.options, nlp, s.emp)
)

MPB.LinearQuadraticModel(s::JAMSDSolver) = JAMSDLinearQuadraticModel(
    JAMSDMathProgModel(s.solver_name, s.options, qcp, s.emp)
)

function MPB.ConicModel(s::JAMSDSolver)
    error("ConicModel is not yet supported")
end

function MPB.loadproblem!(outer::JAMSDNonlinearModel, nvar::Integer, ncon::Integer,
                      x_l, x_u, g_l, g_u, sense::Symbol,
                      d::MPB.AbstractNLPEvaluator)
    m = outer.inner

    m.nvar, m.ncon = nvar, ncon
    loadcommon!(m, x_l, x_u, g_l, g_u, sense)

    m.d = d

    MPB.initialize(m.d.value, [:ExprGraph])

    # Process constraints
    m.constrs = map(1:m.ncon) do i
        c = MPB.constr_expr(m.d.value, i)

        # Remove relations and bounds from constraint expressions
        if length(c.args) == 3
            if VERSION < v"0.5-"
                expected_head = :comparison
                expr_index = 1
                rel_index = 2
            else
                expected_head = :call
                expr_index = 2
                rel_index = 1
            end

            @assert c.head == expected_head
            # Single relation constraint: expr rel bound
            rel = c.args[rel_index]
            m.r_codes[i] = relation_to_jamsd[rel]
            if rel == [:<=, :(==)]
                m.g_u[i] = c.args[3]
            end
            if rel in [:>=, :(==)]
                m.g_l[i] = c.args[3]
            end
            c = c.args[expr_index]
        else
            # Double relation constraint: bound <= expr <= bound
            @assert c.head == :comparison
            m.r_codes[i] = relation_to_jamsd[:multiple]
            m.g_u[i] = c.args[5]
            m.g_l[i] = c.args[1]
            c = c.args[3]
        end

        # Convert non-linear expression to non-linear, linear and constant
        c, constant, m.conlinearities[i] = process_expression!(
            c, m.lin_constrs[i], m.varlinearities_con)

        # Update bounds on constraint
        m.g_l[i] -= constant
        m.g_u[i] -= constant

        # Update jacobian counts using the linear constraint variables
        for j in keys(m.lin_constrs[i])
            m.j_counts[j] += 1
        end
        c
    end

    # Process objective
    m.obj = MPB.obj_expr(m.d.value)

    if length(m.obj.args) < 2
        m.obj = 0
    else
        # Convert non-linear expression to non-linear, linear and constant
        m.obj, constant, m.objlinearity = process_expression!(
            m.obj, m.lin_obj, m.varlinearities_obj)

        # Add constant back into non-linear expression
        if constant != 0
            m.obj = add_constant(m.obj, constant)
        end
    end
    m
end

function MPB.loadproblem!(outer::JAMSDLinearQuadraticModel, A::AbstractMatrix,
                      x_l, x_u, c, g_l, g_u, sense)
    m = outer.inner
    m.ncon, m.nvar = size(A)

    loadcommon!(m, x_l, x_u, g_l, g_u, sense)

    # Load A into the linear constraints
    @assert (m.ncon, m.nvar) == size(A)
    load_A!(m, A)
    m.constrs = zeros(m.ncon)  # Dummy constraint expression trees

    # Load c
    for (index, val) in enumerate(c)
        m.lin_obj[index] = val
    end
    # TODO(xhub) see if we can get rid of that
    m.obj = 0  # Dummy objective expression tree

    # Process variables bounds
    for j = 1:m.ncon
        lower = m.g_l[j]
        upper = m.g_u[j]
        if lower == -Inf
            if upper == Inf
                error("Neither lower nor upper bound on constraint $j")
            else # <=
                m.r_codes[j] = 2
            end
        else
            if lower == upper  # ==
                m.r_codes[j] = 0
            elseif upper == Inf # >=
                m.r_codes[j] = 1
            else # lb <= expr <= ub
                m.r_codes[j] = -1
            end
        end
    end
    m
end

function load_A!(m::JAMSDMathProgModel, A::SparseMatrixCSC{Float64})
    for var = 1:A.n, k = A.colptr[var] : (A.colptr[var + 1] - 1)
        m.lin_constrs[A.rowval[k]][var] = A.nzval[k]
        m.j_counts[var] += 1
    end
end

function load_A!(m::JAMSDMathProgModel, A::Matrix{Float64})
    for con = 1:m.ncon, var = 1:m.nvar
        val = A[con, var]
        if val != 0
            m.lin_constrs[A.rowval[k]][var] = A.nzval[k]
            m.j_counts[var] += 1
        end
    end
end

function loadcommon!(m::JAMSDMathProgModel, x_l, x_u, g_l, g_u, sense)
    m.x_l, m.x_u = x_l, x_u
    m.g_l, m.g_u = g_l, g_u
    setsense!(m, sense)

    m.lin_constrs = [Dict{Int, Float64}() for _ in 1:m.ncon]
    m.j_counts = zeros(Int, m.nvar)

    m.r_codes = Vector{Int}(undef, m.ncon)

    m.varlinearities_con = fill(:Lin, m.nvar)
    m.varlinearities_obj = fill(:Lin, m.nvar)
    m.conlinearities = fill(:Lin, m.ncon)
    m.objlinearity = :Lin

    m.vartypes = fill(:Cont, m.nvar)
    m.x_0 = zeros(m.nvar)
end

getvartype(m::JAMSDMathProgModel) = copy(m.vartypes)
function setvartype!(m::JAMSDMathProgModel, cat::Vector{Symbol})
    @assert all(x-> (x in [:Cont,:Bin,:Int,:external]), cat)
    m.vartypes = copy(cat)
end

getsense(m::JAMSDMathProgModel) = m.sense
function setsense!(m::JAMSDMathProgModel, sense::Symbol)
    @assert sense == :Min || sense == :Max
    m.sense = sense
end

setwarmstart!(m::JAMSDMathProgModel, v::Vector{Float64}) = m.x_0 = v

function MPB.addquadconstr!(m::JAMSDLinearQuadraticModel, linearidx, linearval, quadrowidx, quadcolidx, quadval, sense, rhs)
   # we have to do a little translation of the sense here ...
   push!(m.inner.quad_equs, tuple(linearidx,linearval,quadrowidx,quadcolidx,quadval, quad_relation_sense[sense], rhs))
end

function MPB.setquadobj!(m::JAMSDLinearQuadraticModel, rowidx, colidx, quadval)
   m.inner.quad_obj = tuple(rowidx, colidx, quadval)
end

function optimize!(m::JAMSDMathProgModel)
    m.status = :NotSolved
    m.solve_exitcode = -1
    m.solve_result_num = -1
    m.solve_result = "?"
    m.model_result_num = -1
    m.model_result = "?"
    m.solve_message = ""

    # There is no non-linear binary type, only non-linear discrete, so make
    # sure binary vars have bounds in [0, 1]
    for i in 1:m.nvar
        if m.vartypes[i] == :Bin
            if m.x_l[i] < 0
                m.x_l[i] = 0
            end
            if m.x_u[i] > 1
                m.x_u[i] = 1
            end
        end
    end

    m.jamsd_options = jamsd_options_set(m.options)

    if m.emp.hasvalue
       return m.emp.value()
    end

    make_var_index!(m)
    make_con_index!(m)

    # Run solver and save exitcode
    t = time()
    m.jamsd_ctx = create_jamsd_ctx(m)
    jamsd_set_modeltype(m)
    # Solve via gams for now
    m.jamsd_ctx_dest, m.gams_dir = jamsd_setup_gams()

    m.solve_exitcode = jamsd_solve(m.jamsd_ctx, m.jamsd_ctx_dest, m.solver_name)

#    ccall((:print_model, libjamsd), Cint, (Ptr{context},), m.jamsd_ctx)
    m.solve_time = time() - t

    if m.solve_exitcode == 0
        report_results(m)
    else
        println("JAMSD: solver failed with status $(m.solve_exitcode)")
        m.status = :Error
        m.solution = fill(NaN, m.nvar)
        m.solve_result = "failure"
        m.solve_result_num = 999
    end
end

function getconstrduals(m::JAMSDMathProgModel)

    ctx = m.jamsd_ctx

    x = fill(NaN, numconstr(m))

   if has_objective(m)
        offset = m.offset
    else
        offset = m.offset - 1
    end

    for idx in 1:m.ncon
       eidx = m.nonquad_idx[idx] + offset
       x[idx] = ctx_getmultiplierval(ctx, eidx)
    end

    return x
end

function getquadconstrduals(quadm::JAMSDLinearQuadraticModel)

    m = quadm.inner
    ctx = m.jamsd_ctx

    x = fill(NaN, numconstr(m))

    if has_objective(m)
        offset = m.offset
    else
        offset = m.offset - 1
    end

    for (idx, equ) in enumerate(m.quad_equs)
       eidx = m.quad_idx[idx] + offset
       x[idx] = ctx_getmultiplierval(ctx, eidx)
    end

    return x
end

function getreducedcosts(m::JAMSDMathProgModel)

    ctx = m.jamsd_ctx

    x = fill(NaN, numvar(m))

    for idx in 1:numvar(m)
       x[idx] = ctx_getvarmult(ctx, idx-1)
    end

    return x
end

MPB.getreducedcosts(nlpm::JAMSDNonlinearModel) = getreducedcosts(nlpm.inner)
MPB.getreducedcosts(quadm::JAMSDLinearQuadraticModel) = getreducedcosts(quadm.inner)
MPB.getconstrduals(nlpm::JAMSDNonlinearModel) = getconstrduals(nlpm.inner)
MPB.getconstrduals(quadm::JAMSDLinearQuadraticModel) = getconstrduals(quadm.inner)

function process_expression!(nonlin_expr::Expr, lin_expr::Dict{Int, Float64},
                             varlinearities::Vector{Symbol})
    # Get list of all variables in the expression
    extract_variables!(lin_expr, nonlin_expr)
    # Extract linear and constant terms from non-linear expression
    tree = LinearityExpr(nonlin_expr)
    tree = pull_up_constants(tree)
    _, tree, constant = prune_linear_terms!(tree, lin_expr)
    # Make sure all terms remaining in the tree are .nl-compatible
    nonlin_expr = convert_formula(tree)

    # Track which variables appear nonlinearly
    nonlin_vars = Dict{Int, Float64}()
    extract_variables!(nonlin_vars, nonlin_expr)
    for j in keys(nonlin_vars)
        varlinearities[j] = :Nonlin
    end

    # Remove variables at coeff 0 that aren't also in the nonlinear tree
    for (j, coeff) in lin_expr
        if coeff == 0 && !(j in keys(nonlin_vars))
            delete!(lin_expr, j)
        end
    end

    # Mark constraint as nonlinear if anything is left in the tree
    linearity = nonlin_expr != 0 ? :Nonlin : :Lin

    return nonlin_expr, constant, linearity
end
function process_expression!(nonlin_expr::Real, lin_expr, varlinearities)
    # Special case where body of constraint is constant
    # Return empty nonlinear and linear parts, and use the body as the constant
    0, nonlin_expr, :Lin
end

status(m::JAMSDMathProgModel) = m.status
getsolution(m::JAMSDMathProgModel) = copy(m.solution)
getobjval(m::JAMSDMathProgModel) = m.objval
numvar(m::JAMSDMathProgModel) = m.nvar
numconstr(m::JAMSDMathProgModel) = m.ncon + length(m.quad_equs)
getsolvetime(m::JAMSDMathProgModel) = m.solve_time

# Access to solve results
get_solve_result(m::JAMSDMathProgModel) = m.solve_result
get_solve_result_num(m::JAMSDMathProgModel) = m.solve_result_num
get_model_result(m::JAMSDMathProgModel) = m.model_result
get_model_result_num(m::JAMSDMathProgModel) = m.model_result_num
get_solve_message(m::JAMSDMathProgModel) = m.solve_message
get_solve_exitcode(m::JAMSDMathProgModel) = m.solve_exitcode

# We need to track linear coeffs of all variables present in the expression tree
extract_variables!(lin_constr::Dict{Int, Float64}, c) = c
extract_variables!(lin_constr::Dict{Int, Float64}, c::LinearityExpr) =
    extract_variables!(lin_constr, c.c)
function extract_variables!(lin_constr::Dict{Int, Float64}, c::Expr)
    if c.head == :ref
        CONFIG[:debug] && println("DEBUG: extract_variables :: variable case: $c")
        if c.args[1] == :x
            @assert isa(c.args[2], Int)
            lin_constr[c.args[2]] = 0
        else
            error("Unrecognized reference expression $c")
        end
    else
        map(arg -> extract_variables!(lin_constr, arg), c.args)
    end
end

add_constant(c, constant::Real) = c + constant
add_constant(c::Expr, constant::Real) = Expr(:call, :+, c, constant)

function make_var_index!(m::JAMSDMathProgModel)
    m.v_index_map = Dict(zip(1:m.nvar, 0:(m.nvar-1)))
    m.v_index_map_rev = Dict(zip(0:(m.nvar-1), 1:m.nvar))
end

#function make_var_index!(m::JAMSDMathProgModel)
#    nonlin_cont = Int[]
#    nonlin_int = Int[]
#    lin_cont = Int[]
#    lin_int = Int[]
#    lin_bin = Int[]
#
#    # TODO(xhub) we do that multiple times in the EMP context
#    for i in 1:length(m.vartypes)
#        if m.varlinearities_obj[i] == :Nonlin ||
#           m.varlinearities_con[i] == :Nonlin
#            if m.vartypes[i] == :Cont || m.vartypes[i] == :external
#                push!(nonlin_cont, i)
#            else
#                push!(nonlin_int, i)
#            end
#        else
#           if m.vartypes[i] == :Cont || m.vartypes[i] == :external
#                push!(lin_cont, i)
#            elseif m.vartypes[i] == :Int
#                push!(lin_int, i)
#            else
#                push!(lin_bin, i)
#            end
#        end
#    end
#
#    # Index variables in required order
#    for var_list in (nonlin_cont, nonlin_int, lin_cont, lin_bin, lin_int)
#        add_to_index_maps!(m.v_index_map, m.v_index_map_rev, var_list, 0)
#    end
#    CONFIG[:debug] && println("DEBUG: $(m.v_index_map)")
#end

function make_con_index!(m::JAMSDMathProgModel)
    nonlin_cons = Int[]
    lin_cons = Int[]

    for i in 1:m.ncon
        if m.conlinearities[i] == :Nonlin
            push!(nonlin_cons, i)
        else
            push!(lin_cons, i)
        end
    end
    for con_list in (nonlin_cons, lin_cons)
        add_to_index_maps!(m.nonquad_idx, con_list, 1)
    end

    if length(m.quad_idx) == 0
       m.quad_idx = Dict(enumerate(Vector{Int}(Compat.range(1+m.ncon, length=length(m.quad_equs)))))
    end

    CONFIG[:debug] && println("DEBUG: make_con_index: nonquad_idx: $(m.nonquad_idx)\n quad_idx: $(m.quad_idx)")
end

function add_to_index_maps!(forward_map::Dict{Int, Int},
                            backward_map::Dict{Int, Int},
                            inds::Array{Int},
                            offset::Int)
    for i in inds
        index = length(forward_map) + offset
        forward_map[i] = index
        backward_map[index] = i
    end
end

function add_to_index_maps!(forward_map::Dict{Int, Int},
                            inds::Array{Int},
                            offset::Int)
    for i in inds
        index = length(forward_map) + offset
        forward_map[i] = index
    end
end

function report_results_common(m::JAMSDMathProgModel)
    x = fill(NaN, m.nvar)
    m.objval = NaN

    for index in 0:(m.nvar - 1)
        i = m.v_index_map_rev[index]
        x[i] = ctx_getvarval(m.jamsd_ctx, index)
    end

    m.solution = x

    ###########################################################################
    # Convert solve_result
    #
    # GAMS return two information:
    #  - the solve status
    #  - the model status
    #
    # - :Optimal
    # - :Infeasible
    # - :Unbounded
    # - :UserLimit (iteration limit or timeout)
    # - :Error (and maybe others)
    ###########################################################################

    tmpCint = Ref{Cint}(0)
    res = ccall((:ctx_getsolvestat, libjamsd), Cint, (Ptr{context}, Ref{Cint}), m.jamsd_ctx_dest, tmpCint)
    res != 0 && error("return code $res from JAMSD")
    m.solve_result_num = tmpCint.x
    m.solve_result = unsafe_string(ccall((:ctx_getsolvestattxt, libjamsd), Cstring, (Ptr{context}, Cint), m.jamsd_ctx_dest, m.solve_result_num))

    res = ccall((:ctx_getmodelstat, libjamsd), Cint, (Ptr{context}, Ref{Cint}), m.jamsd_ctx_dest, tmpCint)
    res != 0 && error("return code $res from JAMSD")

    m.model_result_num = tmpCint.x

    m.model_result = unsafe_string(ccall((:ctx_getmodelstattxt, libjamsd), Cstring, (Ptr{context}, Cint), m.jamsd_ctx_dest, m.model_result_num))

    # GAMS already uses an 1-indices
    solver_code = solver_stat[m.solve_result_num]
    model_code = model_stat[m.model_result_num]

    CONFIG[:debug] && println("solver stat $(m.solve_result) ($(m.solve_result_num)); model stat $(m.model_result) ($(m.model_result_num))")

    if solver_code == :Optimal
       if model_code == :OptimalGlobal || model_code == :OptimalLocal || model_code == :Integer
          m.status = :Optimal
       elseif model_code == :Unbounded || model_code == :UnboundedNoSolution
          m.status = :Unbounded
       elseif model_code == :InfeasibleGlobal || model_code == :InfeasibleLocal || model_code == :InfeasibleIntermed || model_code == :InfeasibleNoSolution
          m.status = :Infeasible
       elseif model_code == :Feasible
          # TODO investigate that. Baron is weird
          m.status = :Optimal
       elseif model_code == :NoSolutionReturned
          gams_solver = ctx_get_solvername(m.jamsd_ctx_dest)
          m.status = :Optimal
#          if gams_solver == "jams" || gams_solver == "JAMS"
             # This is fine, we have a kludge in the code
#          else
#             println("JAMSD: Solve successed, but no solution was returned by solver $(gams_solver)!")
#             m.status = :Error
#          end
       else
          println("JAMSD: unhandle case: solver stat $(m.solve_result); model stat $(m.model_result)")
          m.status = :Error
       end
    elseif solver_code == :Iteration || solver_code == :Resource
       m.status == :UserLimit
    elseif solver_code == :License || model_code == :LicenseError
       println("JAMSD: License error. Check that you have a valid license")
       m.status == :Error
    elseif solver_code == :Capability
       if m.solver_name == ""
          sname = default
       else
          sname = m.solver_name
       end

       println("JAMSD: solver $(name) cannot solve the specific problem")
       m.status = :Error

    else
       println("JAMSD: solver stat is $(m.solve_result) and model stat is $(m.model_result)")
       m.status = :Error
    end

    CONFIG[:debug] && println("status is $(m.status)")
 end

function report_results(m::JAMSDMathProgModel)
    # TODO(Xhub) fix this hack
    res = ccall((:model_eval_eqns, libjamsd), Cint, (Ptr{context}, Ptr{context}), m.jamsd_ctx, m.jamsd_ctx_dest)
    res != 0 && error("JAMSD: error code $res")

    # Next, read for the variable values
    report_results_common(m)

    # TODO(xhub) this should not be necessary
    if m.solve_exitcode == 0
        if m.objlinearity == :Nonlin
            # Try to use NLPEvaluator if we can.
            # Can fail due to unsupported functions so fallback to eval
            try
                m.objval = eval_f(m.d.value, m.solution)
            catch
                CONFIG[:debug] && println("Error: could not evaluate the objective function")
            end
        end

        # Calculate objective value from nonlinear and linear parts
        obj_nonlin = eval(substitute_vars!(deepcopy(m.obj), m.solution))
        obj_lin = evaluate_linear(m.lin_obj, m.solution)
        if (length(m.quad_obj) == 3)
           ridx, cidx, vals = m.quad_obj
           obj_quad = evaluate_quad(ridx, cidx, vals, m.solution)
        else
           obj_quad = 0.
        end
        m.objval = obj_nonlin + obj_lin + obj_quad
    end
end

substitute_vars!(c, x::Array{Float64}) = c
function substitute_vars!(c::Expr, x::Array{Float64})
    if c.head == :ref
        if c.args[1] == :x
            index = c.args[2]
            @assert isa(index, Int)
            c = x[index]
        else
            error("Unrecognized reference expression $c")
        end
    else
        if c.head == :call
            # Convert .nl unary minus (:neg) back to :-
            if c.args[1] == :neg
                c.args[1] = :-
            # Convert .nl :sum back to :+
            elseif c.args[1] == :sum
                c.args[1] = :+
            end
        end
        map!(arg -> substitute_vars!(arg, x), c.args, c.args)
    end
    c
end

function evaluate_linear(linear_coeffs::Dict{Int, Float64}, x::Array{Float64})
    total = 0.0
    for (i, coeff) in linear_coeffs
        total += coeff * x[i]
    end
    total
end

function evaluate_quad(rowidx, colidx, qvals, x::Array{Float64})
   n = length(x)
   mat = sparse(rowidx, colidx, qvals, n, n)
   # This is soooo ugly --xhub
   Q = (mat + mat') - Diagonal(diag(mat))
   total = .5*x'*Q*x
   return total
end

# Wrapper functions
for f in [:getvartype,:getsense,:optimize!,:status,:getsolution,:getobjval,:numvar,:numconstr,:getsolvetime]
    @eval MPB.$f(m::JAMSDNonlinearModel) = $f(m.inner)
    @eval MPB.$f(m::JAMSDLinearQuadraticModel) = $f(m.inner)
end

for f in [:get_solve_result,:get_solve_result_num,:get_solve_message,:get_solve_exitcode]
    @eval $f(m::JAMSDNonlinearModel) = $f(m.inner)
    @eval $f(m::JAMSDLinearQuadraticModel) = $f(m.inner)
end
for f in [:setvartype!,:setsense!,:setwarmstart!]
    @eval MPB.$f(m::JAMSDNonlinearModel, x) = $f(m.inner, x)
    @eval MPB.$f(m::JAMSDLinearQuadraticModel, x) = $f(m.inner, x)
end

# Utility method for deleting any leftover debug files
function clean_solverdata()
    for file in readdir(solverdata_dir)
        ext = splitext(file)[2]
        (ext == ".nl" || ext == ".sol") && rm(joinpath(solverdata_dir, file))
    end
end

include("jamsd_mathprgm.jl")
include("jamsd_ovf.jl")
include("jamsd_solve.jl")

function jamsd_cleanup(o::JAMSDMathProgModel)
    ctx_dealloc(o.jamsd_ctx)
    ctx_dealloc(o.jamsd_ctx_dest)
    jamsd_options_dealloc(o.jamsd_options)
    if (!isempty(o.gams_dir))
        try
            rm(o.gams_dir, recursive=true, force=true)
        catch
            iswin && run(`cmd /C RMDIR /s /q $(o.gams_dir)`)
        end
    end
end

function jamsd_options_set(opt::Dict{String,Any})
    jopt = jamsd_options_alloc()

    for (k,v) in opt
        jamsd_option_set(jopt, k, v)
    end

    return jopt
end

function jamsd_setemp!(m::JAMSDSolver, emp)
   m.emp = emp
end

end


using DrWatson
@quickactivate("HyperspectraWithNeXL")
# Load the necessary libraries
using HyperspectraWithNeXL
using NeXLSpectrum
using Gadfly # , Cairo, Fontconfig,
using Unitful
# Load the HyperSpectrum from disk
lt = 0.72*4.0*18.0*3600.0/(1024*1024) # 18.0 hours on 4 detectors
hs = NeXLSpectrum.compress(HyperSpectrum(
    LinearEnergyScale(0.0,10.0),
    Dict{Symbol,Any}(
      :TakeOffAngle => deg2rad(35.0),
      :ProbeCurrent => 1.0,
      :LiveTime => lt,
      :BeamEnergy => 20.0e3,
      :Name => "Mn Nodule"
    ),
    readrplraw(joinpath(datadep"MnNodule","map[15]")),
    fov = [ 4.096u"mm", 4.096u"mm"], offset= [ 0.0u"mm", 0.0u"mm" ]
))
klms = [
    n"C K-L2", #
    characteristic(n"Ag", ltransitions, 0.1)..., #
    n"Al K-L3", #
    n"Ca K-L3", n"Ca K-M3", #
    n"Ce L3-M5", #
    n"Cl K-L3", #
    n"Cr K-L3", n"Cr K-M3", n"Cr L3-M5", #
    n"Cu K-L3", n"Cu K-M3", n"Cu L3-M5", #
    n"Fe K-L3", n"Fe K-M3", n"Fe L3-M5", #
    n"S K-L3", n"S K-M3", #
    characteristic(n"Ba", ltransitions, 0.03)..., #
    n"P K-L3", #
    n"K K-L3", #
    n"Mg K-L3", #
    n"O K-L3", #
    n"Mn K-L3", n"Mn K-M3", n"Mn L3-M5", #
    n"Na K-L3", #
    n"Ni K-L3", n"Ni K-M3", n"Ni L3-M5", #
    n"Si K-L3", #
    n"Ti K-L3", n"Ti K-M3", n"Ti L3-M4", #
    n"Zn K-L3", n"Zn K-M3", n"Zn L3-M5", #
]
set_default_plot_size(8inch, 3inch)
p1 = vstack(
    plot(maxpixel(hs), klms=klms, xmax=10.0e3),
    plot(sum(hs), klms=klms, xmax=10.0e3)
)
# p1 |> PDF(joinpath(plotsdir(),"MaxPixel.pdf"), 6inch, 4inch)
p1 |> SVG(joinpath(plotsdir(), "MaxPixel.svg"), 6inch, 4inch)

p2=plot(maxpixel(hs), sum(hs), klms=klms, xmax=10.0e3, norm=ScaleSum())
# p2 |> PDF(joinpath(plotsdir(),"MaxPixel2.pdf"), 8inch, 3inch)
p2 |> SVG(joinpath(plotsdir(), "MaxPixel2.svg"), 6inch, 4inch)

mkpath(joinpath(papersdir(),"Figures", "Figure 2"))
p3=plot(maxpixel(hs), klms=klms, xmax=10.0e3)
p3 |> SVG(joinpath(papersdir(),"Figures", "Figure 2","maxpixel.svg"), 8inch, 3inch)