function IRNSSAlmanac(arg0::jint, arg1::jint, arg10::jdouble, arg11::jdouble, arg12::AbsoluteDate, arg2::jdouble, arg3::jdouble, arg4::jdouble, arg5::jdouble, arg6::jdouble, arg7::jdouble, arg8::jdouble, arg9::jdouble)
    return IRNSSAlmanac((jint, jint, jdouble, jdouble, AbsoluteDate, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble), arg0, arg1, arg10, arg11, arg12, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9)
end

function get_af0(obj::IRNSSAlmanac)
    return jcall(obj, "getAf0", jdouble, ())
end

function get_af1(obj::IRNSSAlmanac)
    return jcall(obj, "getAf1", jdouble, ())
end

function get_cic(obj::IRNSSAlmanac)
    return jcall(obj, "getCic", jdouble, ())
end

function get_cis(obj::IRNSSAlmanac)
    return jcall(obj, "getCis", jdouble, ())
end

function get_crc(obj::IRNSSAlmanac)
    return jcall(obj, "getCrc", jdouble, ())
end

function get_crs(obj::IRNSSAlmanac)
    return jcall(obj, "getCrs", jdouble, ())
end

function get_cuc(obj::IRNSSAlmanac)
    return jcall(obj, "getCuc", jdouble, ())
end

function get_cus(obj::IRNSSAlmanac)
    return jcall(obj, "getCus", jdouble, ())
end

function get_date(obj::IRNSSAlmanac)
    return jcall(obj, "getDate", AbsoluteDate, ())
end

function get_e(obj::IRNSSAlmanac)
    return jcall(obj, "getE", jdouble, ())
end

function get_i0(obj::IRNSSAlmanac)
    return jcall(obj, "getI0", jdouble, ())
end

function get_i_dot(obj::IRNSSAlmanac)
    return jcall(obj, "getIDot", jdouble, ())
end

function get_iodec(obj::IRNSSOrbitalElements)
    return jcall(obj, "getIODEC", jint, ())
end

function get_m0(obj::IRNSSAlmanac)
    return jcall(obj, "getM0", jdouble, ())
end

function get_mean_motion(obj::IRNSSAlmanac)
    return jcall(obj, "getMeanMotion", jdouble, ())
end

function get_omega0(obj::IRNSSAlmanac)
    return jcall(obj, "getOmega0", jdouble, ())
end

function get_omega_dot(obj::IRNSSAlmanac)
    return jcall(obj, "getOmegaDot", jdouble, ())
end

function get_pa(obj::IRNSSAlmanac)
    return jcall(obj, "getPa", jdouble, ())
end

function get_prn(obj::IRNSSAlmanac)
    return jcall(obj, "getPRN", jint, ())
end

function get_sma(obj::IRNSSAlmanac)
    return jcall(obj, "getSma", jdouble, ())
end

function get_tgd(obj::IRNSSOrbitalElements)
    return jcall(obj, "getTGD", jdouble, ())
end

function get_time(obj::IRNSSAlmanac)
    return jcall(obj, "getTime", jdouble, ())
end

function get_week(obj::IRNSSAlmanac)
    return jcall(obj, "getWeek", jint, ())
end


# vim:set ft=julia ts=4 sw=4 sts=4 autoindent:

include("LoadEnvironment.jl")

require("Corpora.jl")
require("CVMAbstract.jl")
require("CVMAddition.jl")
require("CVMVariance.jl")
require("CVMMean.jl")
require("CVMBigramAddition.jl")
require("CVMBigramMean.jl")
require("CommonTools.jl")
require("DataTypes.jl")
require("Optimizers.jl")

# using ProfileView
using DataTypes
using ArrayViews
using CVMAbstractModel
using Addition
using Mean
using BigramAddition
using BigramMean
using Variance
using CommonTools.CVMIntermediates
using Optimizers
using CommonTools.@nogc

using Corpora: Token, Sentence
using Corpora: DummyToken, DataToken
using Corpora: text, read_linecorpus, replace_vocab!
using JSON
using ArgParse

function parse_commandline()
    s = ArgParseSettings()

    @add_arg_table s begin
        "path_l1"
            help = "Path for l1 training data (sentence aligned to l2!!)"
            required = true
        "path_l2"
            help = "Path for l2 training data (sentence aligned to l1!!)"
            required = true
        "path_l1_mono"
            help = "Path for monolingual l1 training data"
            required = true
        "path_l2_mono"
            help = "Path for monolingual l2 training data"
            required = true
        "path_output"
            help = "Path to store the model and representations after training."
            required = true
        "--pre_trained_l1"
            help = "Path to pre-trained word representations for language 1."
            arg_type = String
            default = ""
        "--pre_trained_l2"
            help = "Path to pre-trained word representations for language 2."
            arg_type = String
            default = ""
        "--modeltype"
            help = "Type of model to train. (composition function)"
            arg_type = String
            default = "Addition"
        "--optimizer"
            help = "Optimizer: Choose from 'AdaGrad', 'RMSProp', 'AdaDelta'"
            arg_type = String
            default = "AdaGrad"
        "--wordvecdim"
            help = "Dimensionality of the word vectors"
            arg_type = Int
            default = 40
        "--margin"
            help = "Hinge Loss margin"
            arg_type = Float
            default = float(40.0)
        "--nnoise"
            help = "Number of noise samples"
            arg_type = Int
            default = 1
        "--batchsize"
            help = "Size of batches over which the gradient will be accumulated"
            arg_type = Int
            default = 40000
        "--regularizer"
            help = "L2 regularizer influence"
            arg_type = Float
            default = float(0.2)
        "--gamma"
            help = "Influence of the noise contrastive objective (in contrast to the 'keep it close' objective"
            arg_type = Float
            default = float(1.0)
        "--bi_closeness_scaling"
            help = "Influence of closeness objective"
            arg_type = Float
            default = float(1.0)
        "--initial_lr"
            help = "Initial learning rate (will be decreased during training)"
            arg_type = Float
            default = float(0.2)
        "--minimal_lr"
            help = "Minimal learning rate"
            arg_type = Float
            default = float(0.2)
        "--rms_decay"
            help = "Decay for RMSProp or AdaDelta(between 0.0 and 1.0)"
            arg_type = Float
            default = float(0.99)
        "--niter"
            help = "Number of iterations"
            arg_type = Int
            default = 100
        "--limit_lines_mono"
            help = "Only read in the specified number of lines in the monolingual case"
            arg_type = Int
            default = 0
        "--unk"
            help = "Unk token"
            default = "_UNK_"
        "--skipunk"
            help = "Skip unknown tokens instead of replacing them by the unk"
            action = :store_true
        "--sparse_updates"
            help = "Only add regularization to gradients that are non-zero"
            action = :store_true
        "--bilingual_contrastive"
            help = "Instead of bilingual closeness, apply the contrastive criterion as used in Hermann et al. (2014)"
            action = :store_true
        "--debug"
            help = "Turn on debug mode (working with less data)"
            action = :store_true
    end

    return parse_args(s)
end

function dump_to_file(model, vocab1, vocab2, path, args)
    """
        Dumps the given model to disk
    """

    # Save model
    open(joinpath(path,"model"),"w") do fout
        d = JSON.parse(JSON.json(model))
        d["modeltype"] = args["modeltype"]
        d["path_l1"] = args["path_l1"]
        d["path_l2"] = args["path_l2"]
        d["path_l1_mono"] = args["path_l1_mono"]
        d["path_l2_mono"] = args["path_l2_mono"]
        write(fout,JSON.json(d))
    end
    # Save vocabulary l1
    open(joinpath(path,"wordvecs.l1"),"w") do fout
        println(fout , model.wordvecdim, " ", length(vocab1))
        for tok in sort(collect(values(vocab1)), by=x->x.count[1], rev=true)
            string_vec = join(map(x->(@sprintf "%0.8f" x),tok.vector), " ")
            println(fout ,tok.word, " ", string_vec)
        end
    end
    # Save vocabulary l2
    open(joinpath(path,"wordvecs.l2"),"w") do fout
        println(fout , model.wordvecdim, " ", length(vocab2))
        for tok in sort(collect(values(vocab2)), by=x->x.count[1], rev=true)
            string_vec = join(map(x->(@sprintf "%0.8f" x),tok.vector), " ")
            println(fout ,tok.word, " ", string_vec)
        end
    end
end

function load_wordvecs(path_wordvecs::String)
    """
        Load pre-trained word vectors from disk
    """
    d = Dict{String,Token}()
    open(path_wordvecs) do fin
        dim, nvecs = map(int64, split(strip(readline(fin))," "))
        for line in eachline(fin)
            s = split(strip(line)," ")
            d[s[1]] = Token(s[1], map(float,s[2:end]),
                                ones(dim), [0])
        end
    end
    return d;
end

function main()
    # Parse Command line
    args = parse_commandline()

    # Create folder and log file
    mkpath(args["path_output"])
    logfile = open(joinpath(args["path_output"],"progress.log"),"w")

    # Dump arguments
    open(joinpath(args["path_output"],"args"),"w") do fout
        write(fout,JSON.json(args))
    end

    # Save the current state of the code together with the model
    dump_code_path = abspath(joinpath(args["path_output"],"code"))
    mkpath(dump_code_path)
    current_dir = splitdir(@__FILE__)[1]
    # Note: This will only work on systems that have find installed
    # (=> not Windows)
    run(`find $current_dir -maxdepth 1 -name "*\.jl"` |>
                `xargs -i cp {} $dump_code_path`)

    # Initialize model according to the user's selection
    if args["modeltype"] == "BigramMean"
       global ModelModule = BigramMean
       typealias Model BigramMean.BigramMeanModel
       typealias Parameters BigramMean.BigramMeanParameters
    elseif args["modeltype"] == "BigramAddition"
       global ModelModule = BigramAddition
       typealias Model BigramAddition.BigramAdditionModel
       typealias Parameters BigramAddition.BigramAdditionParameters
    elseif args["modeltype"] == "Addition"
       global ModelModule = Addition
       typealias Model Addition.AdditionModel
       typealias Parameters Addition.AdditionParameters
    elseif args["modeltype"] == "Mean"
       global ModelModule = Mean
       typealias Model Mean.MeanModel
       typealias Parameters Mean.MeanParameters
    elseif args["modeltype"] == "Variance"
       global ModelModule = Variance
       typealias Model Variance.VarianceModel
       typealias Parameters Variance.VarianceParameters
    end

    ## Some statics
    # RMSProp/AdaDelta decay
    rms_decay = float(args["rms_decay"])
    @assert 0.0 <= rms_decay <= 1.0
    # Influence of bilingual closeness objective
    # The bigger, the more emphasis we put on the bilingual objective
    bi_closeness_scaling = args["bi_closeness_scaling"]
    # Only regularize parameters that have non-zero gradients
    sparse_updates = args["sparse_updates"]
    # Instead of applying bilingual closeness, use the
    # bilingual noise contrastive criterion in Hermann et al. (2014)
    bilingual_contrastive = args["bilingual_contrastive"]
    # Debug mode with smaller number of samples
    line_limit = args["debug"] ? 10000 : 0
    line_limit_mono = args["debug"] ? 10000 : 0
    if args["limit_lines_mono"] > 0
        line_limit_mono = args["limit_lines_mono"]
    end

    # Initialize model
    model = Model(args["wordvecdim"], # word vector dimensionality
                    args["margin"], # Margin
                    args["regularizer"], #l2 regularizer influence
                    args["initial_lr"], # initial learning rate
                    args["minimal_lr"], # minimal learning rate
                    args["niter"], # number of iterations
                    args["gamma"], # weighting of the hinge objective
                    args["nnoise"], # number of noise samples
                    args["batchsize"]) # size of gradient batches

    # If the user has specified a path to pre-trained vectors, load them
    vocab_l1 = Dict{String,Token}()
    if !isempty(args["pre_trained_l1"])
        vocab_l1 = load_wordvecs(args["pre_trained_l1"])
    end
    vocab_l2 = Dict{String,Token}()
    if !isempty(args["pre_trained_l2"])
        vocab_l2 = load_wordvecs(args["pre_trained_l2"])
    end

    # Initializes a large parameter array for the given vocabulary size
    function init_params(nvocab::Int64, params::AbstractParameters)
        params.wordvecs = randn(model.wordvecdim, nvocab) * 0.1
        params.gradients = zeros(model.wordvecdim, nvocab)
    end

    # Creates a DataToken from a given DataToken
    # Usually needed when we load pre-trained word vectors
    function init_from_data(iword::Int64, tok::DataToken,
                                        params::AbstractParameters)
        return DataToken(tok.word, iword, tok.count, tok.vector, tok.gradient)
    end

    ## tricking Julia here a bit
    # Create a vector that doesn't hold a copy but only a reference
    # to the big parameter array
    # This way, we have everything nicely encapsulated in the
    # Token struct and don't need to fiddle with indices anymore
    function vec_from_pointer(p::Ptr{Float}, iword, width)
        byteshift = sizeof(Float) * (iword-1) * width
        return pointer_to_array(p+byteshift, (width,))
    end

    # Initializes a data token from a dummy token
    # Dummy tokens are used a placeholders when we load all the data.
    # After loading the whole data we know the vocabulary sizes and
    # can initialize the large arrays holding all the parameters.
    # Once that is done, we replace DummyTokens by DataTokens which
    # then hold references to the actual word vectors
    function init_from_dummy(iword::Int64, tok::DummyToken,
                                            params::AbstractParameters)
        return DataToken(tok.word, iword, tok.count,
                            vec_from_pointer(pointer(params.wordvecs),
                                                    iword, model.wordvecdim),
                            vec_from_pointer(pointer(params.gradients),
                                                    iword, model.wordvecdim))
    end

    ##### Load language 1 part of the bilingual corpus
    params_l1 = Parameters()
    (data_l1, vocab_l1_bi,
        ntokens_l1, max_len_l1) = read_linecorpus(args["path_l1"],
                                    (x, y) -> init_from_data(x,y,params_l1),
                                    (x, y) -> init_from_dummy(x,y,params_l1),
                                    x -> init_params(x, params_l1),
                                    limit_lines=line_limit,
                                    skiprows=0, dummy_vocab=vocab_l1,
                                    dummies_only=true);

    ##### Load language 2 part of the bilingual corpus
    params_l2 = Parameters()
    (data_l2, vocab_l2_bi,
        ntokens_l2, max_len_l2) = read_linecorpus(args["path_l2"],
                                    (x, y) -> init_from_data(x,y,params_l2),
                                    (x, y) -> init_from_dummy(x,y,params_l2),
                                    x -> init_params(x, params_l2),
                                    limit_lines=line_limit,
                                    skiprows=0, dummy_vocab=vocab_l2,
                                    dummies_only=true);

    println(logfile, "[Only Bilingual]: [L1] Vocab size: $(length(vocab_l1))")
    println(logfile, "[Only Bilingual]: [L2] Vocab size: $(length(vocab_l2))")

    ##### Load monolingual data for language 1
    tic()
    (data_mono_l1, vocab_l1,
        ntokens_mono_l1, max_len_mono_l1) = read_linecorpus(args["path_l1_mono"],
                                    (x, y) -> init_from_data(x,y,params_l1),
                                    (x, y) -> init_from_dummy(x,y,params_l1),
                                    x -> init_params(x, params_l1),
                                    limit_lines=line_limit_mono,
                                    dummy_vocab=vocab_l1_bi,
                                    skiprows=0);

    ##### Load monolingual data for language 2
    (data_mono_l2, vocab_l2,
        ntokens_mono_l2, max_len_mono_l2) = read_linecorpus(args["path_l2_mono"],
                                    (x, y) -> init_from_data(x,y,params_l2),
                                    (x, y) -> init_from_dummy(x,y,params_l2),
                                    x -> init_params(x, params_l2),
                                    limit_lines=line_limit_mono,
                                    dummy_vocab=vocab_l2_bi,
                                    skiprows=0);
    println("Time to read corpus: $(toq()) seconds")
    tic()
    ##### Replace the dummy tokens by real data tokens
    replace_vocab!(data_l1, vocab_l1)
    replace_vocab!(data_l2, vocab_l2)
    println("Time to replace vocab: $(toq()) seconds")

    # Check if the data from both languages has the same size
    # If it doesn't, it can't be sentence aligned
    @assert size(data_l1) == size(data_l2)
    nsentences = size(data_l1,1)
    nsentences_mono_l1 = size(data_mono_l1,1)
    nsentences_mono_l2 = size(data_mono_l2,1)

    # Vocabulary sizes
    n_vocab_l1 = length(vocab_l1)
    n_vocab_l2 = length(vocab_l2)

    # Vocabulary lists
    words_l1 = collect(values(vocab_l1))
    words_l2 = collect(values(vocab_l2))

    # Create Optimizer for l1 and l2
    (opt_l1, opt_l2) = if args["optimizer"] == "AdaGrad"
        (AdaGrad(params_l1.wordvecs, params_l1.gradients),
         AdaGrad(params_l2.wordvecs, params_l2.gradients))
    elseif args["modeltype"] == "RMSProp"
        (RMSProp(params_l1.wordvecs, params_l1.gradients, args["rms_decay"]),
         RMSProp(params_l2.wordvecs, params_l2.gradients, args["rms_decay"]))
    elseif args["modeltype"] == "AdaDelta"
        (AdaDelta(params_l1.wordvecs, params_l1.gradients, args["rms_decay"]),
         AdaDelta(params_l2.wordvecs, params_l2.gradients, args["rms_decay"]))
    end

    # Initialize objectes for intermedate values
    # => since garbage collection is slow, we don't want to re-create them
    # for every run of the inner loop.
    # These structures simply store intermediate values (forward pass results).
    # CAREFUL: In case of implementing parallelism later these intermediate
    # values need to be taken into account. Each thread/process needs its own
    # set of intermediates.
    intermediates = CVMIntermediates[CVMIntermediates(model.wordvecdim,
                                 max(max_len_l1, max_len_l2,
                                     max_len_mono_l1, max_len_mono_l2)),
                     CVMIntermediates(model.wordvecdim,
                                 max(max_len_l1, max_len_l2,
                                     max_len_mono_l1, max_len_mono_l2)),
                     CVMIntermediates(model.wordvecdim,
                                 max(max_len_l1, max_len_l2,
                                    max_len_mono_l1, max_len_mono_l2))]

    println(logfile, "Loading Data completed")
    println(logfile, "Bilingual")
    println(logfile, "\tL1")
    println(logfile, "\t\tNumber of sentences: $(size(data_l1,1))")
    println(logfile, "\t\tNumber of tokens: $(ntokens_l1)")
    println(logfile, "\t\tVocabulary size: $(length(vocab_l1))")
    println(logfile, "\tL2")
    println(logfile, "\t\tNumber of sentences: $(size(data_l2,1))")
    println(logfile, "\t\tNumber of tokens: $(ntokens_l2)")
    println(logfile, "\t\tVocabulary size: $(length(vocab_l2))")
    println(logfile, "Monolingual")
    println(logfile, "\tL1")
    println(logfile, "\t\tNumber of sentences: $(size(data_mono_l1,1))")
    println(logfile, "\t\tNumber of tokens: $(ntokens_mono_l1)")
    println(logfile, "\tL2")
    println(logfile, "\t\tNumber of sentences: $(size(data_mono_l2,1))")
    println(logfile, "\t\tNumber of tokens: $(ntokens_mono_l2)")

    #########################################################################
    ######################## Training #######################################
    #########################################################################

    println(logfile, "##############")
    println(logfile, "Starting training")
    # Outer and inner windows of the inclusion criterion will always be between
    # the specified boundaries
    minmaxwindow_outer = (10, 50);
    minwindow_inner = 3;
    minmaxwindown = (10, 50);
    ### Perform the training
    # The total number of samples that we process is calculated
    # as the number of sentences in the largest data collection (mono_l1,
    # mono_l2 or bi) times the number of iterations specified by the user
    max_nsentences = max(nsentences, nsentences_mono_l1, nsentences_mono_l2)
    nsentences_total = max_nsentences * model.niter
    isentence_total = 0
    # Some varibles to store accumulated errors
    acc_total = const_zero;
    acc_closeness_bi = const_zero;
    acc_contr_mono_l1 = const_zero;
    acc_closeness_mono_l1 = const_zero;
    acc_contr_mono_l2 = const_zero;
    acc_closeness_mono_l2 = const_zero;
    acc_reg = const_zero;
    isentence_tic = const_zero;
    tic();
    # Learning rate (initially) (will be decreased according to specified
    # parameters)
    lr = model.lr_init;

    ################## START THE TRAINING #######################
    println("Starting training")
    #= Profile.clear() =#
    while isentence_total < nsentences_total
        # Since we always iterate over a full batch,
        # we might end up training for slightly more than the specified
        # iterations.
        # Turn off the garbage collection while processing a batch
        @nogc for ibatch=1:model.batchsize
            # Calculate the index of the current bilingual sentence pair
            isentence = (isentence_total % nsentences) + 1
            # Adaptive learning rate
            lr = float(max(model.lr_min,
                        model.lr_init *
                            (const_one - isentence_total / nsentences_total)))

            # Select the text of the current sentence and its translation
            s1 = data_l1[isentence].text
            s2 = data_l2[isentence].text
            # Make sure all sentences have a certain minimal length
            if (size(s1,1) < 5 || size(s2,1) < 5)
                isentence_total += 1
                continue
            end

            # Collect the gradients for the bilingual criterion.
            # All the gradients are stored in the corresponding field
            # of each token. This field is a reference to the large gradient
            # array the contains all gradients.
            if bilingual_contrastive
                sn = data_l2[rand(1:nsentences)].text
                err_closeness_bi, err_contr_bi = accumulate_gradients_batch!(
                                                model,
                                                s1, s2, Vector{Token}[sn],
                                                intermediates,
                                                scaling=bi_closeness_scaling)
                acc_closeness_bi += err_closeness_bi + err_contr_bi
            else
                acc_closeness_bi += accumulate_closeness_gradients_batch!(model,
                                                s1, s2,
                                                intermediates,
                                                scaling=bi_closeness_scaling)
            end

            #### Monolingual
            for (data, ns, lbl) in ((data_mono_l1, nsentences_mono_l1, 1),
                            (data_mono_l2, nsentences_mono_l2, 2))
                # Calculate the running index for the current
                # sentence in the current corpus
                icursentence = (isentence_total%ns)+1
                # Get the sentence length and check if it
                # is longer than the specified minimum
                lensen = size(data[icursentence].text,1)
                if lensen < minmaxwindow_outer[1]+2
                    continue
                end

                ### Outer window
                # Calculate the offset for the valid sentence
                offset_outer = rand(1:lensen-1-minmaxwindow_outer[1])
                # Get the end of the window
                # |-----|...............|.........|------------------|
                #    offset         min_winend   max_winend    sentence_end
                min_winend_outer = offset_outer+minmaxwindow_outer[1]
                window_end_outer = rand(min_winend_outer:
                                            min(min_winend_outer +
                                                minmaxwindow_outer[2],
                                                lensen))
                ### Inner window
                # Length of the sub sentence
                lensubsen = window_end_outer - offset_outer
                # Get the offset for the inner window
                offset_inner = rand(1:lensubsen-1-minwindow_inner)
                # Earliest end of inner window
                min_winend_inner = offset_inner+minwindow_inner
                # Get a random inner window end that is bigger than the minsize
                window_end_inner = rand(min_winend_inner:lensubsen)

                ### Noise (same procedure as for outer window)
                # Calculate the offset for the valid sentence
                noise_samples = Vector{Token}[]
                while size(noise_samples,1) < model.nnoise
                    # Get the index for the noise sentence
                    inoise = icursentence
                    # Make sure the noise sentence is not equivalent
                    # to the current valid sentence
                    while inoise == icursentence
                        inoise = rand(1:ns)
                    end
                    # Check if the length of the noise sentence
                    # is above the specified threshold
                    lensen_noise = size(data[inoise].text,1)
                    if lensen_noise < minmaxwindown[1]+2
                        continue
                    end

                    offsetn = rand(1:lensen_noise-1-minmaxwindown[1])
                    min_winendn = offsetn+minmaxwindown[1]
                    window_endn = rand(min_winendn:
                                                min(min_winendn +
                                                    minmaxwindown[2],
                                                    lensen_noise))
                    sn = data[inoise].text[offsetn:window_endn]
                    push!(noise_samples,sn)
                end

                ### Retrieve the sentences with the previously calculated indices
                s1 = data[icursentence].text[offset_outer:window_end_outer]
                s2 = s1[offset_inner:window_end_inner]
                # Calculate the overlap ratio of the inner and outer window
                overlap_ratio = size(s2,1)/size(s1,1);
                # Accumulate the gradients for the inclusion objective
                err_closeness, err_contr = accumulate_gradients_batch!(model,
                                                s1, s2, noise_samples,
                                                intermediates,
                                                scaling=overlap_ratio)
                # Update the accumulative error
                if lbl == 1
                    acc_contr_mono_l1 += err_contr
                    acc_closeness_mono_l1 += err_closeness
                elseif lbl == 2
                    acc_contr_mono_l2 += err_contr
                    acc_closeness_mono_l2 += err_closeness
                else
                    throw(ErrorException("Unknown label given for acc errors."))
                end
            end

            # Increase the sentence counter
            isentence_total += 1
        end

        # Apply the squared l2 regularizer
        acc_reg += regularize_l2!(model, params_l1,
                                        sparse_updates=sparse_updates)
        acc_reg += regularize_l2!(model, params_l2,
                                        sparse_updates=sparse_updates)
        # Execute one update step of the optimizers
        # The optimizer also resets all the gradients accumulated in the
        # last batch to zero
        step!(opt_l1, lr)
        step!(opt_l2, lr)

        # Calculate the total accumulated error
        acc_total = (acc_closeness_bi + acc_reg + acc_contr_mono_l1 +
                        acc_closeness_mono_l1 + acc_contr_mono_l2 +
                        acc_closeness_mono_l2)

        # Print some progress statistics
        time_elapsed = toq();
        @printf logfile "Progress: %0.2f%% " (isentence_total)/nsentences_total*100
        @printf logfile "lr: %0.5f " lr
        @printf logfile "error: total %0.1f " (acc_total/model.batchsize)
        @printf logfile "closeness %0.3f " (acc_closeness_bi/model.batchsize)
        @printf logfile "cl-mono-l1 %0.3f " (acc_closeness_mono_l1/model.batchsize)
        @printf logfile "co-mono-l1 %0.3f " (acc_contr_mono_l1/model.batchsize/model.nnoise)
        @printf logfile "cl-mono-l2 %0.3f " (acc_closeness_mono_l2/model.batchsize)
        @printf logfile "co-mono-l2 %0.3f " (acc_contr_mono_l2/model.batchsize/model.nnoise)
        @printf logfile "reg %0.3f " (acc_reg/model.batchsize)
        @printf logfile "sents/s: %6d \n" ((isentence_total-isentence_tic)/time_elapsed)
        flush(logfile)
        isentence_tic = isentence_total
        # Reset the accumulative errors
        acc_total = const_zero;
        acc_contr_mono_l1 = const_zero;
        acc_closeness_mono_l1 = const_zero;
        acc_contr_mono_l2 = const_zero;
        acc_closeness_mono_l2 = const_zero;
        acc_closeness_bi = const_zero;
        acc_reg = const_zero;
        # Rest the timer that we use to estimate number-sentences/sec
        tic()
    end

    ##################### TRAINING FINISHED #####################

    # Create the output path if it doesn't exist
    dump_to_file(model, vocab_l1, vocab_l2, args["path_output"],
                    args);

    # Close the log file
    close(logfile)

end

main()


using LAMMPS

function lammpssnapdescriptors(x, t, a, b, c, pbc, unitstyle, pair_style, pair_coeff, snapparam, elemradius, elemweight)

    lmp = LMP()

    # handle boundary conditions
    bcs = ""
    for i = 1:3
        if pbc[i] == 1
            bcs = bcs * " p"
        else
            bcs = bcs * " f"
        end
    end

    command(lmp, "units " * unitstyle)
    command(lmp, "boundary " * bcs)
    command(lmp, "atom_style atomic")
    command(lmp, "atom_modify map array")
    command(lmp, "box tilt large")

    # create simulation box
    xlo = 0.0; ylo = 0.0; zlo = 0.0;
    xhi = a[1]; yhi = b[2]; zhi = c[3];
    xy = b[1]; xz = c[1]; yz = c[2];
    if (abs(xy) + abs(xz) + abs(yz)) <= 1e-10
        command(lmp, "region simbox block $xlo $xhi $ylo $yhi $zlo $zhi units box")
    else
        command(lmp, "region simbox prism $xlo $xhi $ylo $yhi $zlo $zhi $xy $xz $yz units box")
    end
    command(lmp, "create_box 1 simbox")

    # pair potential
    command(lmp, "pair_style " * pair_style)
    for i = 1:length(pair_coeff)
        command(lmp, "pair_coeff " * pair_coeff[i])
    end

    # set mass
    ntypes = length(unique(t))
    for i = 1:ntypes
        itype = Int32(i)
        command(lmp, "mass $itype 1.0")
    end

    # create atom types and atom positions
    natom = size(x,2)
    id = Int32.(Array(1:(natom)))
    v = 0.0*x
    image = zeros(Int32, natom)
    bexpand = 0
    LAMMPS.API.lammps_create_atoms(lmp, natom, id, Int32.(t), x, v, image, bexpand)

    command(lmp, "compute PE all pe")
    command(lmp, "compute S all pressure thermo_temp")

    # snap parammeters
    ntypes = Int32(snapparam[1]);
    twojmax = Int32(snapparam[2]);
    rcutfac = snapparam[3];
    rfac0 = snapparam[4];
    rmin0 = snapparam[5];
    bzeroflag = Int32(snapparam[6]);
    switchflag = Int32(snapparam[7]);
    quadraticflag = Int32(snapparam[8]);
    chemflag = Int32(snapparam[9]);
    bnormflag = Int32(snapparam[10]);
    wselfallflag = Int32(snapparam[11]);
    flags = " bzeroflag $bzeroflag quadraticflag $quadraticflag switchflag $switchflag bnormflag $bnormflag wselfallflag $wselfallflag"

    radius = ""
    weight = ""
    nelements = length(elemradius)
    for i = 1:nelements
        elemr = elemradius[i]
        elemw = elemweight[i]
        radius = radius * " $elemr"
        weight = weight * " $elemw"
    end
    keys = " rmin0 $rmin0"
    if (nelements > 1) & (chemflag==1)
        keys = keys * " chem $nelements"
        for i = 1:nelements
            itype = Int32(i)
            keys = keys * " $itype"
        end
    end

    # compute snap descriptors
    command(lmp, "compute snap all snap $rcutfac $rfac0 $twojmax" * radius * weight * flags * keys)

    # run lammps
    command(lmp, "thermo_style custom pe")
    command(lmp, "run 0")

    # all : [num_bispectrum + num_reference] x [num_energy + num_forces + num_virials]
    all = extract_compute(lmp, "snap", LAMMPS.API.LMP_STYLE_GLOBAL,
                                       LAMMPS.API.LMP_TYPE_ARRAY)

    # extract bispectrum, derivatives, and virials
    bi = all[1:end-1,1]
    bd = all[1:end-1,2:(1+3*natom)]
    bv = all[1:end-1,(2+3*natom):end];

    num_bispectrum = length(bi)
    bd = reshape(bd, (num_bispectrum, 3, natom))
    bd = permutedims(bd, (2,3,1))
    bv = permutedims(bv, (2,1))

    # extract reference energy, forces, and stresses
    energy = all[end,1]
    forces = reshape(all[end,2:(1+3*natom)], (3, natom))
    stresses = all[end,(2+3*natom):end];

    return bi, bd, bv, energy, forces, stresses
end


struct GenericOpProblem <: PowerSimulationsOperationsProblem end

mutable struct OperationsProblem{M <: AbstractOperationsProblem}
    template::OperationsProblemTemplate
    sys::PSY.System
    psi_container::PSIContainer
end

"""
    OperationsProblem(::Type{M},
    template::OperationsProblemTemplate,
    sys::PSY.System,
    jump_model::Union{Nothing, JuMP.AbstractModel}=nothing;
    kwargs...) where {M<:AbstractOperationsProblem,
                      T<:PM.AbstractPowerFormulation}
This builds the optimization problem of type M with the specific system and template.
# Arguments
- `::Type{M} where M<:AbstractOperationsProblem`: The abstract operation model type
- `template::OperationsProblemTemplate`: The model reference made up of transmission, devices,
                                          branches, and services.
- `sys::PSY.System`: the system created using Power Systems
- `jump_model::Union{Nothing, JuMP.AbstractModel}`: Enables passing a custom JuMP model. Use with care
# Output
- `op_problem::OperationsProblem`: The operation model containing the model type, built JuMP model, Power
Systems system.
# Example
```julia
template = OperationsProblemTemplate(CopperPlatePowerModel, devices, branches, services)
OpModel = OperationsProblem(TestOpProblem, template, system)
```
# Accepted Key Words
- `horizon::Int`: Manually specify the length of the forecast Horizon
- `initial_time::Dates.DateTime`: Initial Time for the model solve
- `use_forecast_data::Bool` : If true uses the data in the system forecasts. If false uses the data for current operating point in the system.
- `PTDF::PTDF`: Passes the PTDF matrix into the optimization model for StandardPTDFModel networks.
- `optimizer::JuMP.MOI.OptimizerWithAttributes`: The optimizer that will be used in the optimization model.
- `use_parameters::Bool`: True will substitute will implement formulations using ParameterJuMP parameters. Defatul is false.
- `warm_start::Bool` True will use the current operation point in the system to initialize variable values. False initializes all variables to zero. Default is true
- `balance_slack_variables::Bool` True will add slacks to the system balance constraints
- `services_slack_variables::Bool` True will add slacks to the services requirement constraints
"""
function OperationsProblem(
    ::Type{M},
    template::OperationsProblemTemplate,
    sys::PSY.System,
    jump_model::Union{Nothing, JuMP.AbstractModel} = nothing;
    kwargs...,
) where {M <: AbstractOperationsProblem}
    return OperationsProblem{M}(template, sys, jump_model; kwargs...)
end

function OperationsProblem{M}(
    template::OperationsProblemTemplate,
    sys::PSY.System,
    jump_model::Union{Nothing, JuMP.AbstractModel} = nothing;
    kwargs...,
) where {M <: AbstractOperationsProblem}
    check_kwargs(kwargs, OPERATIONS_ACCEPTED_KWARGS, "OperationsProblem")
    settings = PSISettings(sys; kwargs...)
    return OperationsProblem{M}(template, sys, jump_model, settings)
end

# The psi_container_init is called at the build! call in this constructor. This is meant to
# build an operation from a template.
function OperationsProblem{M}(
    template::OperationsProblemTemplate,
    sys::PSY.System,
    jump_model::Union{Nothing, JuMP.AbstractModel},
    settings::PSISettings,
) where {M <: AbstractOperationsProblem}
    op_problem =
        OperationsProblem{M}(template, sys, PSIContainer(sys, settings, jump_model))
    build!(op_problem)
    return op_problem
end

"""
    OperationsProblem(op_problem::Type{M},
                    ::Type{T},
                    sys::PSY.System,
                    jump_model::Union{Nothing, JuMP.AbstractModel}=nothing;
                    kwargs...) where {M<:AbstractOperationsProblem,
                                      T<:PM.AbstractPowerFormulation}
Return an unbuilt operation problem of type M with the specific system and network model T.
    This constructor doesn't build any device model; it is meant to built device models individually using [`construct_device!`](@ref)
# Arguments
- `::Type{M} where M<:AbstractOperationsProblem`: The abstract operation model type
- `::Type{T} where T<:AbstractPowerModel`: The abstract network formulation
- `sys::PSY.System`: the system created using Power Systems
- `jump_model::Union{Nothing, JuMP.AbstractModel}`: Enables passing a custom JuMP model. Use with care
# Output
- `op_problem::OperationsProblem`: The operation model containing the model type, unbuilt JuMP model, Power
Systems system.
# Example
```julia
OpModel = OperationsProblem(MyCustomOpProblem, DCPPowerModel, system)
model = DeviceModel(ThermalStandard, ThermalStandardUnitCommitment)
construct_device!(op_problem, :Thermal, model)
```
# Accepted Key Words
- `horizon::Int`: Manually specify the length of the forecast Horizon
- `initial_time::Dates.DateTime`: Initial Time for the model solve
- `use_forecast_data::Bool` : If true uses the data in the system forecasts. If false uses the data for current operating point in the system.
- `PTDF::PTDF`: Passes the PTDF matrix into the optimization model for StandardPTDFModel networks.
- `optimizer::JuMP.MOI.OptimizerWithAttributes`: The optimizer that will be used in the optimization model.
- `use_parameters::Bool`: True will substitute will implement formulations using ParameterJuMP parameters. Defatul is false.
- `warm_start::Bool` True will use the current operation point in the system to initialize variable values. False initializes all variables to zero. Default is true
- `balance_slack_variables::Bool` True will add slacks to the system balance constraints
- `services_slack_variables::Bool` True will add slacks to the services requirement constraints
"""
function OperationsProblem(
    ::Type{M},
    ::Type{T},
    sys::PSY.System,
    jump_model::Union{Nothing, JuMP.AbstractModel} = nothing;
    kwargs...,
) where {M <: AbstractOperationsProblem, T <: PM.AbstractPowerModel}
    return OperationsProblem{M}(T, sys, jump_model; kwargs...)
end

"""
    OperationsProblem(::Type{T},
                    sys::PSY.System,
                    jump_model::Union{Nothing, JuMP.AbstractModel}=nothing;
                    kwargs...) where {M<:AbstractOperationsProblem,
                                      T<:PM.AbstractPowerFormulation}
Return an unbuilt operation problem of type GenericOpProblem with the specific system and network model T.
    This constructor doesn't build any device model; it is meant to built device models individually using [`construct_device!`](@ref)
# Arguments
- `::Type{T} where T<:AbstractPowerModel`: The abstract network formulation
- `sys::PSY.System`: the system created using Power Systems
- `jump_model::Union{Nothing, JuMP.AbstractModel}`: Enables passing a custom JuMP model. Use with care
# Output
- `op_problem::OperationsProblem`: The operation model containing the model type, unbuilt JuMP model, Power
Systems system.
# Example
```julia
OpModel = OperationsProblem(DCPPowerModel, system)
model = DeviceModel(ThermalStandard, ThermalStandardUnitCommitment)
construct_device!(op_problem, :Thermal, model)
```
# Accepted Key Words
- `horizon::Int`: Manually specify the length of the forecast Horizon
- `initial_time::Dates.DateTime`: Initial Time for the model solve
- `use_forecast_data::Bool` : If true uses the data in the system forecasts. If false uses the data for current operating point in the system.
- `PTDF::PTDF`: Passes the PTDF matrix into the optimization model for StandardPTDFModel networks.
- `optimizer::JuMP.MOI.OptimizerWithAttributes`: The optimizer that will be used in the optimization model.
- `use_parameters::Bool`: True will substitute will implement formulations using ParameterJuMP parameters. Defatul is false.
- `warm_start::Bool` True will use the current operation point in the system to initialize variable values. False initializes all variables to zero. Default is true
- `balance_slack_variables::Bool` True will add slacks to the system balance constraints
- `services_slack_variables::Bool` True will add slacks to the services requirement constraints
"""
function OperationsProblem(
    ::Type{T},
    sys::PSY.System,
    jump_model::Union{Nothing, JuMP.AbstractModel} = nothing;
    kwargs...,
) where {T <: PM.AbstractPowerModel}
    return OperationsProblem{GenericOpProblem}(T, sys, jump_model; kwargs...)
end

# This constructor calls PSI container including the the PowerModels type in order to initialize
# the container and it is meant to construct operations problems using construct_device! function
# calls
function OperationsProblem{M}(
    ::Type{T},
    sys::PSY.System,
    jump_model::Union{Nothing, JuMP.AbstractModel} = nothing;
    kwargs...,
) where {M <: AbstractOperationsProblem, T <: PM.AbstractPowerModel}
    check_kwargs(kwargs, OPERATIONS_ACCEPTED_KWARGS, "OperationsProblem")
    settings = PSISettings(sys; kwargs...)
    return OperationsProblem{M}(
        OperationsProblemTemplate(T),
        sys,
        PSIContainer(T, sys, settings, jump_model),
    )

end

"""
    OperationsProblem(filename::AbstractString)

Construct an OperationsProblem from a serialized file.

# Arguments
- `filename::AbstractString`: path to serialized file
- `jump_model::Union{Nothing, JuMP.AbstractModel}` = nothing: The JuMP model does not get
   serialized. Callers should pass whatever they passed to the original problem.
- `optimizer::Union{Nothing,JuMP.MOI.OptimizerWithAttributes}` = nothing: The optimizer does
   not get serialized. Callers should pass whatever they passed to the original problem.
"""
function OperationsProblem(
    filename::AbstractString;
    jump_model::Union{Nothing, JuMP.AbstractModel} = nothing,
    optimizer::Union{Nothing, JuMP.MOI.OptimizerWithAttributes} = nothing,
)
    return deserialize(
        OperationsProblem,
        filename;
        jump_model = jump_model,
        optimizer = optimizer,
    )
end

get_transmission_ref(op_problem::OperationsProblem) = op_problem.template.transmission
get_devices_ref(op_problem::OperationsProblem) = op_problem.template.devices
get_branches_ref(op_problem::OperationsProblem) = op_problem.template.branches
get_services_ref(op_problem::OperationsProblem) = op_problem.template.services
get_system(op_problem::OperationsProblem) = op_problem.sys
get_psi_container(op_problem::OperationsProblem) = op_problem.psi_container
get_base_power(op_problem::OperationsProblem) = PSY.get_base_power(op_problem.sys)
get_jump_model(op_problem::OperationsProblem) = get_jump_model(op_problem.psi_container)

function reset!(op_problem::OperationsProblem)
    op_problem.psi_container =
        PSIContainer(op_problem.sys, op_problem.psi_container.settings, nothing)
    return
end

function set_transmission_model!(
    op_problem::OperationsProblem{M},
    transmission::Type{T},
) where {T <: PM.AbstractPowerModel, M <: AbstractOperationsProblem}
    op_problem.template.transmission = transmission
    reset!(op_problem)
    build!(op_problem)
    return
end

function set_devices_template!(
    op_problem::OperationsProblem{M},
    devices::Dict{Symbol, DeviceModel},
) where {M <: AbstractOperationsProblem}
    op_problem.template.devices = devices
    reset!(op_problem)
    build!(op_problem)
    return
end

function set_branches_template!(
    op_problem::OperationsProblem{M},
    branches::Dict{Symbol, DeviceModel},
) where {M <: AbstractOperationsProblem}
    op_problem.template.branches = branches
    reset!(op_problem)
    build!(op_problem)
    return
end

function set_services_template!(
    op_problem::OperationsProblem{M},
    services::Dict{Symbol, <:ServiceModel},
) where {M <: AbstractOperationsProblem}
    op_problem.template.services = services
    reset!(op_problem)
    build!(op_problem)
    return
end

function set_device_model!(
    op_problem::OperationsProblem{M},
    name::Symbol,
    device::DeviceModel{<:PSY.StaticInjection, <:AbstractDeviceFormulation},
) where {M <: AbstractOperationsProblem}
    if haskey(op_problem.template.devices, name)
        op_problem.template.devices[name] = device
        reset!(op_problem)
        build!(op_problem)
    else
        throw(IS.ConflictingInputsError("Device Model with name $(name) doesn't exist in the model"))
    end
    return
end

function set_branch_model!(
    op_problem::OperationsProblem{M},
    name::Symbol,
    branch::DeviceModel{<:PSY.Branch, <:AbstractDeviceFormulation},
) where {M <: AbstractOperationsProblem}
    if haskey(op_problem.template.branches, name)
        op_problem.template.branches[name] = branch
        reset!(op_problem)
        build!(op_problem)
    else
        throw(IS.ConflictingInputsError("Branch Model with name $(name) doesn't exist in the model"))
    end
    return
end

function set_services_model!(
    op_problem::OperationsProblem{M},
    name::Symbol,
    service::ServiceModel,
) where {M <: AbstractOperationsProblem}
    if haskey(op_problem.template.services, name)
        op_problem.template.services[name] = service
        reset!(op_problem)
        build!(op_problem)
    else
        throw(IS.ConflictingInputsError("Branch Model with name $(name) doesn't exist in the model"))
    end
    return
end

function set_model!(
    ::Type{D},
    op_problem::OperationsProblem,
    name::Symbol,
    device_model::DeviceModel,
) where {D <: PSY.Branch}
    op_problem.template.branches[name] = device_model
end

function set_model!(
    ::Type{D},
    op_problem::OperationsProblem,
    name::Symbol,
    device_model::DeviceModel,
) where {D <: PSY.StaticInjection}
    set_model!(op_problem.template, name, device_model)
end

function set_model!(
    ::Type{D},
    op_problem::OperationsProblem,
    name::Symbol,
    device_model::DeviceModel,
) where {D <: PSY.Service}
    op_problem.template.services[name] = device_model
end

function construct_device!(
    op_problem::OperationsProblem,
    name::Symbol,
    device_model::DeviceModel,
)
    set_model!(device_model.device_type, op_problem, name, device_model)

    construct_device!(
        op_problem.psi_container,
        get_system(op_problem),
        device_model,
        get_transmission_ref(op_problem),
    )
    JuMP.@objective(
        op_problem.psi_container.JuMPmodel,
        MOI.MIN_SENSE,
        op_problem.psi_container.cost_function
    )
    return
end

function construct_network!(op_problem::OperationsProblem)
    construct_network!(op_problem, op_problem.template.transmission)
    return
end

function construct_network!(
    op_problem::OperationsProblem,
    system_formulation::Type{T},
) where {T <: PM.AbstractPowerModel}
    construct_network!(op_problem.psi_container, get_system(op_problem), T)
    return
end

function get_initial_conditions(
    op_problem::OperationsProblem,
    ic::InitialConditionType,
    device::PSY.Device,
)
    psi_container = op_problem.psi_container
    key = ICKey(ic, device)

    return get_initial_conditions(psi_container, key)
end

function build!(op_problem::OperationsProblem{M}) where {M <: AbstractOperationsProblem}
    sys = get_system(op_problem)
    _build!(op_problem.psi_container, op_problem.template, sys)
    return
end

function check_problem_size(psi_container::PSIContainer)
    vars = JuMP.num_variables(psi_container.JuMPmodel)
    cons = 0
    for (exp, c_type) in JuMP.list_of_constraint_types(psi_container.JuMPmodel)
        cons += JuMP.num_constraints(psi_container.JuMPmodel, exp, c_type)
    end
    return "The current total number of variables is $(vars) and total number of constraints is $(cons)"
end

function _build!(
    psi_container::PSIContainer,
    template::OperationsProblemTemplate,
    sys::PSY.System,
)
    transmission = template.transmission
    # Order is required
    # The container is initialized here because this build! call for psi_container takes the
    # information from the template with cached PSISettings. It allows having the same build! call for operations problems
    # specified with template and simulation stage.
    psi_container_init!(psi_container, transmission, sys)
    construct_services!(psi_container, sys, template.services, template.devices)
    for device_model in values(template.devices)
        @debug "Building $(device_model.device_type) with $(device_model.formulation) formulation"
        construct_device!(psi_container, sys, device_model, transmission)
        @debug check_problem_size(psi_container)
    end
    @debug "Building $(transmission) network formulation"
    construct_network!(psi_container, sys, transmission)
    @debug check_problem_size(psi_container)

    for branch_model in values(template.branches)
        @debug "Building $(branch_model.device_type) with $(branch_model.formulation) formulation"
        construct_device!(psi_container, sys, branch_model, transmission)
        @debug check_problem_size(psi_container)
    end

    @debug "Building Objective"
    JuMP.@objective(psi_container.JuMPmodel, MOI.MIN_SENSE, psi_container.cost_function)
    @debug "Total operation count $(psi_container.JuMPmodel.operator_counter)"
    return
end

function get_variables_value(op_m::OperationsProblem)
    results_dict = Dict{Symbol, DataFrames.DataFrame}()
    for (k, v) in get_variables(op_m.psi_container)
        results_dict[k] = axis_array_to_dataframe(v)
    end
    return results_dict
end

function get_dual_values(op_m::OperationsProblem)
    return get_dual_values(op_m.psi_container)
end

"""
    solve!(op_problem::OperationsProblem; kwargs...)
This solves the operational model for a single instance and
outputs results of type OperationsProblemResult
# Arguments
- `op_problem::OperationModel = op_problem`: operation model
# Examples
```julia
results = solve!(OpModel)
```
# Accepted Key Words
- `save_path::String`: If a file path is provided the results
automatically get written to feather files
- `optimizer::MOI.OptimizerWithAttributes`: The optimizer that is used to solve the model
"""
function solve!(
    op_problem::OperationsProblem{T};
    kwargs...,
) where {T <: AbstractOperationsProblem}
    check_kwargs(kwargs, OPERATIONS_SOLVE_KWARGS, "Solve")
    timed_log = Dict{Symbol, Any}()
    save_path = get(kwargs, :save_path, nothing)

    if op_problem.psi_container.JuMPmodel.moi_backend.state == MOIU.NO_OPTIMIZER
        if !(:optimizer in keys(kwargs))
            error("No Optimizer has been defined, can't solve the operational problem")
        end
        JuMP.set_optimizer(op_problem.psi_container.JuMPmodel, kwargs[:optimizer])
        _,
        timed_log[:timed_solve_time],
        timed_log[:solve_bytes_alloc],
        timed_log[:sec_in_gc] = @timed JuMP.optimize!(op_problem.psi_container.JuMPmodel)
    else
        _,
        timed_log[:timed_solve_time],
        timed_log[:solve_bytes_alloc],
        timed_log[:sec_in_gc] = @timed JuMP.optimize!(op_problem.psi_container.JuMPmodel)
    end
    model_status = JuMP.primal_status(op_problem.psi_container.JuMPmodel)
    if model_status != MOI.FEASIBLE_POINT::MOI.ResultStatusCode
        error("The Operational Problem $(T) status is $(model_status)")
    end
    vars_result = get_variables_value(op_problem)
    param_values = get_parameters_value(get_psi_container(op_problem))
    optimizer_log = get_optimizer_log(op_problem)
    time_stamp = get_time_stamps(op_problem)
    time_stamp = shorten_time_stamp(time_stamp)
    base_power = PSY.get_base_power(op_problem.sys)
    dual_result = get_dual_values(op_problem)
    obj_value = Dict(
        :OBJECTIVE_FUNCTION => JuMP.objective_value(op_problem.psi_container.JuMPmodel),
    )
    base_power = get_base_power(op_problem)
    merge!(optimizer_log, timed_log)

    results = OperationsProblemResults(
        base_power,
        vars_result,
        obj_value,
        optimizer_log,
        time_stamp,
        dual_result,
        param_values,
    )

    !isnothing(save_path) && write_results(results, save_path)

    return results
end

# Function to create a dictionary for the optimizer log of the simulation
function get_optimizer_log(op_m::OperationsProblem)
    psi_container = op_m.psi_container
    optimizer_log = Dict{Symbol, Any}()
    optimizer_log[:obj_value] = JuMP.objective_value(psi_container.JuMPmodel)
    optimizer_log[:termination_status] = JuMP.termination_status(psi_container.JuMPmodel)
    optimizer_log[:primal_status] = JuMP.primal_status(psi_container.JuMPmodel)
    optimizer_log[:dual_status] = JuMP.dual_status(psi_container.JuMPmodel)
    optimizer_log[:solver] = JuMP.solver_name(psi_container.JuMPmodel)

    try
        optimizer_log[:solve_time] = MOI.get(psi_container.JuMPmodel, MOI.SolveTime())
    catch
        @warn("SolveTime() property not supported by $(optimizer_log[:solver])")
        optimizer_log[:solve_time] = "Not Supported by $(optimizer_log[:solver])"
    end
    return optimizer_log
end

# Function to create a dictionary for the time series of the simulation
function get_time_stamps(op_problem::OperationsProblem)
    initial_time = model_initial_time(get_psi_container(op_problem))
    interval = PSY.get_forecasts_resolution(op_problem.sys)
    horizon = get_horizon(get_settings(get_psi_container(op_problem)))
    range_time = collect(initial_time:interval:(initial_time + interval .* horizon))
    time_stamp = DataFrames.DataFrame(Range = range_time[:, 1])

    return time_stamp
end

function write_data(psi_container::PSIContainer, save_path::AbstractString; kwargs...)
    file_type = get(kwargs, :file_type, Feather)
    if file_type == Feather || file_type == CSV
        for (k, v) in get_variables(psi_container)
            file_path = joinpath(save_path, "$(k).$(lowercase("$file_type"))")
            variable = axis_array_to_dataframe(v)
            if isempty(variable)
                @debug "$(k) is empty, not writing $file_path"
            else
                file_type.write(file_path, variable)
            end
        end
    end
    return
end

function write_data(op_problem::OperationsProblem, save_path::String; kwargs...)
    write_data(op_problem.psi_container, save_path; kwargs...)
    return
end

""" Exports the OpModel JuMP object in MathOptFormat"""
function export_op_model(op_problem::OperationsProblem, save_path::String)
    _write_psi_container(op_problem.psi_container, save_path)
    return
end

################ Functions to debug optimization models#####################################
""" "Each Tuple corresponds to (con_name, internal_index, moi_index)"""
function get_all_constraint_index(op_problem::OperationsProblem)
    con_index = Vector{Tuple{Symbol, Int, Int}}()
    for (key, value) in op_problem.psi_container.constraints
        for (idx, constraint) in enumerate(value)
            moi_index = JuMP.optimizer_index(constraint)
            push!(con_index, (key, idx, moi_index.value))
        end
    end
    return con_index
end

""" "Each Tuple corresponds to (con_name, internal_index, moi_index)"""
function get_all_var_index(op_problem::OperationsProblem)
    var_index = Vector{Tuple{Symbol, Int, Int}}()
    for (key, value) in op_problem.psi_container.variables
        for (idx, variable) in enumerate(value)
            moi_index = JuMP.optimizer_index(variable)
            push!(var_index, (key, idx, moi_index.value))
        end
    end
    return var_index
end

function get_con_index(op_problem::OperationsProblem, index::Int)
    for i in get_all_constraint_index(op_problem::OperationsProblem)
        if i[3] == index
            return op_problem.psi_container.constraints[i[1]].data[i[2]]
        end
    end

    @info "Index not found"
    return
end

function get_var_index(op_problem::OperationsProblem, index::Int)
    for i in get_all_var_index(op_problem::OperationsProblem)
        if i[3] == index
            return op_problem.psi_container.variables[i[1]].data[i[2]]
        end
    end
    @info "Index not found"
    return
end

function serialize(op_problem::OperationsProblem, filename::AbstractString)
    # A PowerSystem cannot be serialized in this format because of how it stores
    # time series data. Use its specialized serialization method instead.
    sys_filename = "$(basename(filename))-system-$(IS.get_uuid(op_problem.sys)).json"
    PSY.to_json(op_problem.sys, sys_filename)
    obj = OperationsProblemSerializationWrapper(
        op_problem.template,
        sys_filename,
        op_problem.psi_container.settings_copy,
        typeof(op_problem),
    )
    Serialization.serialize(filename, obj)
    @info "Serialized OperationsProblem to" filename
end

function deserialize(::Type{OperationsProblem}, filename::AbstractString; kwargs...)
    obj = Serialization.deserialize(filename)
    if !(obj isa OperationsProblemSerializationWrapper)
        throw(IS.DataFormatError("deserialized object has incorrect type $(typeof(obj))"))
    end

    if !ispath(obj.sys)
        throw(IS.DataFormatError("PowerSystems.System file $(obj.sys) does not exist"))
    end
    sys = PSY.System(obj.sys)

    return obj.op_problem_type(
        obj.template,
        sys,
        kwargs[:jump_model],
        restore_from_copy(obj.settings; optimizer = kwargs[:optimizer]),
    )
end

struct OperationsProblemSerializationWrapper
    template::OperationsProblemTemplate
    sys::String
    settings::PSISettings
    op_problem_type::DataType
end


for func in [:clblasSrotm, :clblasDrotm]
    @eval @blasfun $func(N::Csize_t,
                              X::cl.CL_mem, offx::Csize_t, incx::Cint,
                              Y::cl.CL_mem, offy::Csize_t, incy::Cint,
                              SPARAM::cl.CL_mem, offSparam::Csize_t,
                              n_queues::cl.CL_uint,
                              queues::Ptr{cl.CL_command_queue},
                              n_events_in_wait_list::cl.CL_uint,
                              event_wait_list::Ptr{cl.CL_event},
                              events::Ptr{cl.CL_event})

    @eval @blasfun2 $func(N::Csize_t,
                               X::cl.CL_mem, offx::Csize_t, incx::Cint,
                               Y::cl.CL_mem, offy::Csize_t, incy::Cint,
                               SPARAM::cl.CL_mem, offSparam::Csize_t)
end


using PSCOPF

using Test
using Dates
using DataStructures

@testset verbose=true  "test_seq_generator" begin

    #=
    INPUT :
        Network := Electric grid description
        TS := target time steps (dates d'intérêt)
        Mode de gestion = mode 1
        ECH := list of horizon points as generated by the generate_ech responsibility
    EXPECTED OUTPUT :
        At each horizon point, a sequence (an ordered list of operations i.e. AbstractRunnable)
    CARE POINTs:
        - generate_sequences assumes TS is sorted and unique ?
        - generate_sequences assumes ECH is sorted and unique ?
    =#

    @testset "mode_1" begin
        network = PSCOPF.Networks.Network()
        ts1 = Dates.DateTime("2015-01-01T11:00:00")
        TS = PSCOPF.create_target_timepoints(ts1)
        mode = PSCOPF.PSCOPF_MODE_1
        ECH = PSCOPF.generate_ech(network, TS, mode)

        sequence = PSCOPF.generate_sequence(network, TS, ECH, mode)

        EXPECTED_OPERATIONS = SortedDict(
            ts1 - Dates.Hour(4)      => [PSCOPF.EnergyMarket, PSCOPF.TSOOutFO],
            ts1 - Dates.Hour(1)      => [PSCOPF.EnergyMarketAtFO, PSCOPF.EnterFO, PSCOPF.TSOInFO],
            ts1 - Dates.Minute(30)   => [PSCOPF.TSOInFO],
            ts1 - Dates.Minute(15)   => [PSCOPF.TSOInFO],
            ts1                      => [PSCOPF.Assessment]
        )

        @test length(sequence) == length(ECH)
        for (ech, steps) in sequence.operations
            for (index,step) in enumerate(steps)
                @test isa(step, EXPECTED_OPERATIONS[ech][index])
            end
        end
    end

    @testset "mode_2" begin
        network = PSCOPF.Networks.Network()
        ts1 = Dates.DateTime("2015-01-01T11:00:00")
        TS = PSCOPF.create_target_timepoints(ts1)
        mode = PSCOPF.PSCOPF_MODE_2
        ECH = PSCOPF.generate_ech(network, TS, mode)

        sequence = PSCOPF.generate_sequence(network, TS, ECH, mode)

        EXPECTED_OPERATIONS = SortedDict(
            ts1 - Dates.Hour(4)      => [PSCOPF.EnergyMarket, PSCOPF.TSOOutFO],
            ts1 - Dates.Hour(1)      => [PSCOPF.EnergyMarketAtFO, PSCOPF.EnterFO, PSCOPF.TSOOutFO],
            ts1 - Dates.Minute(30)   => [PSCOPF.EnergyMarket, PSCOPF.TSOBilevel, PSCOPF.BalanceMarket],
            ts1 - Dates.Minute(15)   => [PSCOPF.EnergyMarket, PSCOPF.TSOBilevel, PSCOPF.BalanceMarket],
            ts1                      => [PSCOPF.Assessment]
        )

        @test length(sequence) == length(ECH)
        for (ech, steps) in sequence.operations
            for (index,step) in enumerate(steps)
                @test isa(step, EXPECTED_OPERATIONS[ech][index])
            end
        end
    end

    @testset "mode_3" begin
        network = PSCOPF.Networks.Network()
        ts1 = Dates.DateTime("2015-01-01T11:00:00")
        TS = PSCOPF.create_target_timepoints(ts1)
        mode = PSCOPF.PSCOPF_MODE_3
        ECH = PSCOPF.generate_ech(network, TS, mode)

        sequence = PSCOPF.generate_sequence(network, TS, ECH, mode)

        EXPECTED_OPERATIONS = SortedDict(
            ts1 - Dates.Hour(4)      => [PSCOPF.EnergyMarket, PSCOPF.TSOOutFO],
            ts1 - Dates.Hour(1)      => [PSCOPF.EnergyMarket, PSCOPF.EnterFO, PSCOPF.TSOBilevel], #FIXME or maybe TSOBilevelAtFO
            ts1 - Dates.Minute(30)   => [PSCOPF.BalanceMarket],
            ts1 - Dates.Minute(15)   => [PSCOPF.BalanceMarket],
            ts1                      => [PSCOPF.Assessment]
        )

        @test length(sequence) == length(ECH)
        for (ech, steps) in sequence.operations
            for (index,step) in enumerate(steps)
                @test isa(step, EXPECTED_OPERATIONS[ech][index])
            end
        end
    end

end


@testset "Load data misspecification" begin
    device_model = DeviceModel(InterruptibleLoad, DispatchablePowerLoad)
    c_sys5 = PSB.build_system(PSITestSystems, "c_sys5")
    info_message = "The data doesn't include devices of type InterruptibleLoad, consider changing the device models"
    model = DecisionModel(MockOperationProblem, DCPPowerModel, c_sys5)
    @test_logs (:info, info_message) match_mode = :any mock_construct_device!(
        model,
        device_model,
    )
    device_model = DeviceModel(PowerLoad, DispatchablePowerLoad)
    warn_message = "The Formulation DispatchablePowerLoad only applies to FormulationControllable Loads, \n Consider Changing the Device Formulation to StaticPowerLoad"
    model = DecisionModel(MockOperationProblem, DCPPowerModel, c_sys5)
    @test_logs (:warn, warn_message) match_mode = :any mock_construct_device!(
        model,
        device_model,
    )
end

@testset "StaticPowerLoad" begin
    models = [StaticPowerLoad, DispatchablePowerLoad, InterruptiblePowerLoad]
    c_sys5_il = PSB.build_system(PSITestSystems, "c_sys5_il")
    networks = [DCPPowerModel, ACPPowerModel]
    for m in models, n in networks
        device_model = DeviceModel(PowerLoad, m)
        model = DecisionModel(MockOperationProblem, n, c_sys5_il)
        mock_construct_device!(model, device_model)
        moi_tests(model, false, 0, 0, 0, 0, 0, false)
        psi_checkobjfun_test(model, GAEVF)
    end
end

@testset "DispatchablePowerLoad DC- PF" begin
    models = [DispatchablePowerLoad]
    c_sys5_il = PSB.build_system(PSITestSystems, "c_sys5_il")
    networks = [DCPPowerModel]
    for m in models, n in networks
        device_model = DeviceModel(InterruptibleLoad, m)
        model = DecisionModel(MockOperationProblem, n, c_sys5_il)
        mock_construct_device!(model, device_model)
        moi_tests(model, false, 24, 0, 24, 0, 0, false)
        psi_checkobjfun_test(model, GAEVF)
    end
end

@testset "DispatchablePowerLoad AC- PF" begin
    models = [DispatchablePowerLoad]
    c_sys5_il = PSB.build_system(PSITestSystems, "c_sys5_il")
    networks = [ACPPowerModel]
    for m in models, n in networks
        device_model = DeviceModel(InterruptibleLoad, m)
        model = DecisionModel(MockOperationProblem, n, c_sys5_il)
        mock_construct_device!(model, device_model)
        moi_tests(model, false, 48, 0, 24, 0, 24, false)
        psi_checkobjfun_test(model, GAEVF)
    end
end

@testset "InterruptiblePowerLoad DC- PF" begin
    models = [InterruptiblePowerLoad]
    c_sys5_il = PSB.build_system(PSITestSystems, "c_sys5_il")
    networks = [DCPPowerModel]
    for m in models, n in networks
        device_model = DeviceModel(InterruptibleLoad, m)
        model = DecisionModel(MockOperationProblem, n, c_sys5_il)
        mock_construct_device!(model, device_model)
        moi_tests(model, false, 48, 0, 24, 0, 0, true)
        psi_checkobjfun_test(model, GAEVF)
    end
end

@testset "InterruptiblePowerLoad AC- PF" begin
    models = [InterruptiblePowerLoad]
    c_sys5_il = PSB.build_system(PSITestSystems, "c_sys5_il")
    networks = [ACPPowerModel]
    for m in models, n in networks
        device_model = DeviceModel(InterruptibleLoad, m)
        model = DecisionModel(MockOperationProblem, n, c_sys5_il)
        mock_construct_device!(model, device_model)
        moi_tests(model, false, 72, 0, 24, 0, 24, true)
        psi_checkobjfun_test(model, GAEVF)
    end
end


function readoutput(calculation::Calculation{Wannier90}, file; kwargs...)
    return wan_read_output(file; kwargs...)
end

#THIS IS THE MOST HORRIBLE FUNCTION I HAVE EVER CREATED!!!
#extracts only atoms with projections
function extract_atoms(atoms_block, proj_block, cell::Mat3, spinors = false)
    if atoms_block.name == :atoms_cart
        cell = Mat3(Matrix(1.0Ang * I, 3, 3))
    end
    projections_ = proj_block.data
    atoms = atoms_block.data
    t_start = 1
    out_ats = Atom[]
    if projections_ !== nothing
        t_ats = Atom[]
        for (proj_at, projs) in projections_
            for (pos_at, pos) in atoms
                for ps in pos
                    if proj_at != pos_at
                        continue
                    end
                    for proj in projs
                        size = spinors ? 2 * length(proj) : length(proj)
                        push!(t_ats,
                              Atom(; name = pos_at, position_cart = cell * ps,
                                   position_cryst = ps,
                                   projections = [Projection(Structures.orbital(proj),
                                                             t_start, t_start + size - 1)]))
                        t_start += size
                    end
                end
            end
        end
        for (pos_at, pos) in atoms
            for ps in pos
                same_ats = Atom[]
                for at in t_ats
                    if at.position_cart == cell * ps
                        push!(same_ats, at)
                    end
                end
                if isempty(same_ats)
                    continue
                end
                if length(same_ats) > 1
                    for at in same_ats[2:end]
                        push!(same_ats[1].projections, at.projections[1])
                    end
                end
                push!(out_ats, same_ats[1])
            end
        end
    else
        for (pos_at, pos) in atoms
            for p in pos
                push!(out_ats,
                      Atom(pos_at; position_cart = cell * p, position_cryst = p,
                           projections = Projection[]))
            end
        end
    end
    return out_ats
end

function extract_structure(cell_block, atoms_block, projections_block,
                           spinors = false)
    if atoms_block == nothing || cell_block == nothing
        return nothing
    end
    if cell_block.option == :bohr
        cell = cell_block.data' .* 1bohr
    else
        cell = cell_block.data' .* 1Ang
    end

    atoms = extract_atoms(atoms_block, projections_block, cell, spinors)
    return Structure(Mat3(cell), atoms)
end

function wan_read_calculation(::Type{T}, f::IO) where {T}
    flags       = Dict{Symbol,Any}()
    data        = InputData[]
    atoms_block = nothing
    cell_block  = nothing
    proj_block  = nothing
    line        = readline(f)
    while !eof(f)
        @label start_label

        if occursin("!", line) ||
           line == "" ||
           occursin("end", lowercase(line)) ||
           occursin("#", line)
            line = readline(f)
            continue
        end

        if occursin("begin", lowercase(line))
            block_name = Symbol(split(lowercase(line))[end])

            if block_name == :projections
                proj_dict = Tuple{Symbol,Array{String,1}}[]
                line      = readline(f)
                while !occursin("end", lowercase(line))
                    if occursin("!", line) || line == ""
                        line = readline(f)
                        continue
                    end
                    if occursin("random", line)
                        proj_block = InputData(:projections, :random, nothing)
                        line = readline(f)
                        break
                    else
                        split_line  = strip_split(line, ':')
                        atom        = Symbol(split_line[1])
                        projections = [proj for proj in strip_split(split_line[2], ';')]
                        push!(proj_dict, (atom, projections))
                        line = readline(f)
                    end
                end
                proj_block = InputData(:projections, :none, proj_dict)
                @goto start_label

            elseif block_name == :kpoint_path
                line = readline(f)
                k_path_array = Array{Tuple{Symbol,Array{T,1}},1}()
                while !occursin("end", lowercase(line))
                    if occursin("!", line) || line == ""
                        line = readline(f)
                        continue
                    end
                    split_line = split(line)
                    push!(k_path_array,
                          (Symbol(split_line[1]), parse_string_array(T, split_line[2:4])))
                    push!(k_path_array,
                          (Symbol(split_line[5]), parse_string_array(T, split_line[6:8])))
                    line = readline(f)
                end
                push!(data, InputData(:kpoint_path, :none, k_path_array))
                @goto start_label

            elseif block_name == :unit_cell_cart
                line = readline(f)
                if length(split(line)) == 1
                    option = Symbol(strip(lowercase(line)))
                    line = readline(f)
                else
                    option = :ang
                end
                cell_param = Matrix{T}(undef, 3, 3)
                for i in 1:3
                    cell_param[i, :] = parse_line(T, line)
                    line = readline(f)
                end
                cell_block = InputData(:unit_cell_cart, option, Mat3(cell_param))
                # line = readline(f)
                @goto start_label

            elseif block_name == :atoms_frac || block_name == :atoms_cart
                line   = readline(f)
                atoms  = Dict{Symbol,Array{Point3{T},1}}()
                option = :ang
                while !occursin("end", lowercase(line))
                    if occursin("!", line) || line == ""
                        line = readline(f)
                        continue
                    end
                    if length(split(line)) == 1
                        option = Symbol(strip(line))
                        line = readline(f)
                        continue
                    end
                    split_line = strip_split(line)
                    atom       = Symbol(split_line[1])
                    position   = Point3(parse_string_array(T, split_line[2:4]))
                    if !haskey(atoms, atom)
                        atoms[atom] = [position]
                    else
                        push!(atoms[atom], position)
                    end
                    line = readline(f)
                end
                atoms_block = InputData(block_name, option, atoms)
                @goto start_label

            elseif block_name == :kpoints
                line     = readline(f)
                k_points = Array{NTuple{3,T},1}()
                while !occursin("end", lowercase(line))
                    if line == ""
                        line = readline(f)
                        continue
                    end
                    push!(k_points, (parse_line(T, line)...,))
                    line = readline(f)
                end
                push!(data, InputData(:kpoints, :none, k_points))
                @goto start_label
            end

        else
            if occursin("mp_grid", line)
                flags[:mp_grid] = parse_string_array(Int, split(line)[end-2:end])
            else
                flag, val = wan_parse_flag_line(line)
                flags[flag] = val
            end
        end
        line = readline(f)
    end
    return (flags=flags, data=data, atoms=atoms_block, cell=cell_block, projections=proj_block)
end

wan_read_calculation(f::IO) = wan_read_calculation(Float64, f)

"""
    wan_read_calculation(filename::String, T=Float64; run=true, exec=Exec(exec="wannier90.x"), structure_name="NoName")

Reads a `Calculation{Wannier90}` and the included `Structure` from a WANNIER90 calculation file.
"""
function wan_read_calculation(filename::String, T = Float64;
                              exec = Exec(; exec = "wannier90.x"), kwargs...)
    flags       = Dict{Symbol,Any}()
    data        = InputData[]
    atoms_block = nothing
    cell_block  = nothing
    proj_block  = nothing
    open(filename, "r") do f
        return flags, data, atoms_block, cell_block, proj_block = wan_read_calculation(T, f)
    end
    structure = extract_structure(cell_block, atoms_block, proj_block,
                                  get(flags, :spinors, false))
    dir, file = splitdir(filename)
    flags[:preprocess] = Calculations.hasflag(exec,
                                              :pp) ? true : false
    return Calculation{Wannier90}(; name = splitext(file)[1], flags = flags,
                                  data = data, exec = exec, kwargs...), structure
end

function wan_parse_array_value(eltyp, value_str)
    value_str = replace(value_str, "," => "")
    spl = split(value_str)
    arr = eltyp[]
    for i in 1:length(spl)
        if spl[i] == "t"
            spl[i] = "true"
        elseif spl[i] == "f"
            spl[i] = "false"
        end
        s = spl[i]
        if occursin("-", s)
            t = split(s, '-')
            append!(arr, collect(parse(eltyp, t[1]):parse(eltyp, t[2])))
        else
            append!(arr, parse(eltyp, s))
        end
    end
    return arr
end

function wan_parse_flag_line(line::String)
    split_line = strip_split(line, '=')
    flag       = Symbol(split_line[1])
    flagtyp    = Calculations.flagtype(Wannier90, flag)
    value      = strip(lowercase(split_line[2]), '.')
    if flagtyp != String
        value = replace(value, "d" => "e")
    end
    if flagtyp <: Vector
        parsed_val = wan_parse_array_value(eltype(flagtyp), value)
    elseif flagtyp == String
        parsed_val = value
    else
        parsed_val = parse.(eltype(flagtyp), split(value))
    end
    if length(parsed_val) == 1
        val = parsed_val[1]
    else
        val = parsed_val
    end
    return flag, val
end

function wan_write_projections(f::IO, atoms::Vector{Atom})
    write(f, "begin projections\n")
    uniats = unique(atoms)
    projs = map(x -> x.projections, uniats)
    if all(isempty.(projs))
        write(f, "random\n")
    else
        for (at, prjs) in zip(uniats, projs)
            if isempty(prjs)
                continue
            end
            write(f, Structures.projections_string(at))
            write(f, "\n")
        end
    end
    return write(f, "end projections\n")
end

"""
    save(calculation::Calculation{Wannier90}, structure, filename::String)

Writes the `Calculation{Wannier90}` and `structure` to a file, that can be interpreted by WANNIER90.
The atoms in the structure must have projections defined.
"""
function save(calculation::Calculation{Wannier90}, structure,
              filename::String)

    projs = vcat(map(structure.atoms) do x
                     ps = x.projections
                     @assert !isempty(ps) "Please first set projections for all atoms in the Structure."
                     return ps
                 end...)

    Structures.sanitize!(projs, Calculations.issoc(calculation))

    open(filename, "w") do f
        for (flag, value) in calculation.flags
            write_flag_line(f, flag, value)
        end
        write(f, "\n")

        if structure != nothing
            write(f, "begin unit_cell_cart\n")
            writedlm(f, ustrip.(structure.cell'))
            write(f, "end unit_cell_cart\n")
            write(f, "\n")
        end
        wan_write_projections(f, structure.atoms)

        write(f, "\n")
        write(f, "begin atoms_frac\n")
        for at in structure.atoms
            pos = round.(at.position_cryst, digits = 5)
            write(f, "$(at.name)  $(pos[1]) $(pos[2]) $(pos[3])\n")
        end
        write(f, "end atoms_frac\n")
        write(f, "\n")

        for block in calculation.data
            write(f, "begin $(block.name)\n")
            if block.name == :kpoint_path
                for i in 1:2:length(block.data)
                    letter1, k_points1 = block.data[i]
                    letter2, k_points2 = block.data[i+1]
                    write(f,
                          "$letter1 $(k_points1[1]) $(k_points1[2]) $(k_points1[3]) $letter2 $(k_points2[1]) $(k_points2[2]) $(k_points2[3])\n")
                end

            elseif block.name == :kpoints
                for k in block.data
                    write(f, "$(k[1]) $(k[2]) $(k[3])\n")
                end
            end
            write(f, "end $(block.name)\n\n")
        end
    end
end

function write_flag_line(f, flag, data, seperator = "=", i = "")
    flagstr = string(flag)
    if flagstr[end-1] == '_' && tryparse(Int, string(flagstr[end])) != nothing
        flagstr = flagstr[1:end-2] * "($(flagstr[end]))"
    end
    write(f, "  $flagstr$i $seperator ")

    if typeof(data) <: Array
        if length(data) % 3 == 0 && eltype(data) != Int
            write(f, "\n")
            for i in 1:3:length(data)
                write(f, "  $(data[i]) $(data[i + 1]) $(data[i + 2])\n")
            end
        else
            write(f, "  $(data[1])")
            for x in data[2:end]
                write(f, " $x")
            end
            write(f, "\n")
        end

    else #this should work for anything singular valued data such as bools, ''s and other types
        write(f, "$data\n")
    end
end

function wan_parse_disentanglement(out, line, f)
    DisTuple = NamedTuple{(:Iter, :Ω_i_1, :Ω_i, :δ, :Time),
                          Tuple{Int,Float64,Float64,Float64,Float64}}
    out[:disentanglement] = DisTuple[]
    for i in 1:4
        readline(f)
    end
    line = strip(readline(f))
    while !isempty(line)
        push!(out[:disentanglement], parse(DisTuple, split(line)[1:end-2]))
        line = strip(readline(f))
    end
end
function wan_parse_wannierise(out, line, f)
    WanTuple = NamedTuple{(:Iter, :δ_spread, :∇RMS, :Spread, :Time),
                          Tuple{Int,Float64,Float64,Float64,Float64}}
    WfcTuple = NamedTuple{(:index, :center, :Spread),Tuple{Int,Point3{Float64},Float64}}
    out[:wannierise] = WanTuple[]
    line = strip(readline(f))
    while !occursin("Final State", line)
        if occursin("CONV", line)
            push!(out[:wannierise], parse(WanTuple, split(line)[1:end-2]))
        end
        line = strip(readline(f))
    end
    out[:final_state] = WfcTuple[]
    line = strip(readline(f))
    while !occursin("Sum", line)
        line = replace(replace(replace(line, "(" => ""), ")" => ""), "," => "")
        push!(out[:final_state], parse(WfcTuple, split(line)[5:end]))
        line = strip(readline(f))
    end
end

const WAN_PARSE_FUNCS = ["Extraction of optimally-connected subspace" => wan_parse_disentanglement,
                         "Initial State" => wan_parse_wannierise]

"""
    wan_read_output(filename::AbstractString; parse_funcs::Vector{Pair{String,Function}}=Pair{String,Function}[])

Reads an outputfile for wannier.
Parsed info:
    :disentanglement,
    :wannierise,
    :final_state,
"""
function wan_read_output(filename::AbstractString;
                         parse_funcs::Vector{<:Pair{String}} = Pair{String}[])
    return parse_file(filename, WAN_PARSE_FUNCS; extra_parse_funcs = parse_funcs)
end

function outfiles(c::Calculation{Wannier90})
    files = [c.name]
    if get(c, :wannier_plot, false)
        push!(files, "UNK")
    end
    return unique(files)
end


"""
`Basics.interpolate()`  ... interpolates a one- or two-dimensional function by different numerical methods, either by value or on a whole grid.

  + `("function: to new radial grid, Grasp92", (F::Array{Float64,1}, oldGrid::Radial.Grid), (G::Void, newGrid::Radial.Grid) )`
    ... to interpolate the (radial) function F from the oldgrid to newgrid, by a call to the Grasp92 Fortran procedures;
        an function G::Array{Float64,1} is returned. **Not yet implemented !**

  + `("function: to new radial grid, trapez rule", (F::Array{Float64,1}, oldGrid::Radial.Grid), (G::Void, newGrid::Radial.Grid)`
    ... to integrate the (radial) function F from the oldgrid to newgrid, by using a trapez rule from the Numerical Recipies.
        A function G::Array{Float64,1} is returned. **Not yet implemented !**
"""
function Basics.interpolate(sa::String, from::Tuple{Array{Float64,1},JAC.Radial.Grid}, to::Tuple{DataType,JAC.Radial.Grid})
    if       sa == "function: to new radial grid, Grasp92"          wa = interpolateOnGridGrasp92(from, to)
    elseif   sa == "function: to new radial grid, trapez rule"      wa = interpolateOnGridTrapezRule(from, to)
    else     error("Unsupported keystring = $sa.")
    end

end

"""
`Basics.interpolateOnGridGrasp92()`  ... interpolates ... by using Grasp92 Fortran procedures; see Basic.interpolate().
"""
function Basics.interpolateOnGridGrasp92(from::Tuple{Array{Float64,1},JAC.Radial.Grid}, to::Tuple{DataType,JAC.Radial.Grid})
    # First prepare all fields and settings to call quad_grasp92()
    error("Not yet implemented !")
end

"""
`Basics.interpolateOnGridTrapezRule()`  ... interpolates ... by using trapez rule from Numerical Recipies; see Basic.interpolate().
"""
function Basics.interpolateOnGridTrapezRule(from::Tuple{Array{Float64,1},JAC.Radial.Grid}, to::Tuple{DataType,JAC.Radial.Grid})
    error("Not yet implemented !")
end


import GR
using Base.Threads
using Distributed

# include(joinpath(dirname(@__FILE__), "kernels.jl"))
# include(joinpath(dirname(@__FILE__), "sph_types.jl"))
# include(joinpath(dirname(@__FILE__), "mapping_functions.jl"))

"""
    glimpse(filename::String, blockname::String[, ... ])

Reads relevant data from snapshot file and maps the quantity in block `blockname`
to a grid.

# Arguments
- `filename::String`: Name of snapshot file.
- `blockname::String`: Name of block that should be mapped.
- `center_pos::Array{Float64,1}=[123456.7, 123456.7, 123456.7]`: Image center. If not given center of mass is calculated.
- `dx::Float64`= 0.0`: Extent in x-direction. If `dx = 0.0` whole box is mapped.
- `dy::Float64`= 0.0`: Extent in y-direction. If `dy = 0.0` whole box is mapped.
- `dz::Float64`= 0.0`: Extent in z-direction. If `dz = 0.0` whole box is mapped.
- `kernel_name::String="WC6"`: Which kernel should be used. ["Cubic", "Quintic", "WC4", "WC6"]
- `resolution::Int64=500`: Number of pixels in the longest dimension.
- `run_dummy::Bool=true`: If a compilation run with 4 pixels should be performed.
- `parallel::Bool=true`: Run on multiple processors.
- `conserve_quantities::Bool=true`: If quantities should be conserved while mapping, like in Smac (Dolag et. al. 2005).
- `verbose::Bool=true`: Output information to console and show progress bar.
- `plot::Bool=false`: Plot the resulting map with GR.imshow()
"""
function glimpse(filename::String, blockname::String,
                 center_pos::Array{Float64,1}=[123456.7, 123456.7, 123456.7],
                 dx::Float64=0.0, dy::Float64=0.0, dz::Float64=0.0;
			     kernel_name::String="WC6",
                 resolution::Int64=500, run_dummy::Bool=true,
				 parallel::Bool=true,
				 conserve_quantities::Bool=false,
			     verbose::Bool=true, plot::Bool=false)

    if verbose
		@info "Reading data..."
    end

	filebase = filename

	# if the snapshot does not exist it may be split into multiple files
    if !isfile(filebase)

        if verbose
            @info "File: $filebase not found, looking for sub-files."
        end

        # try reading the first of the distributed snapshots
        filename = filebase * ".0"

        # throw error if file does not exist
        if !isfile(filename)
            error("File: $filename not found!")
        end

        h = head_to_obj(filename)

        if verbose
            @info "$(h.num_files) sub-files found."
        end

        nfiles = h.num_files
    else
        nfiles = 1
    end

    # read header of snapshot
    h = head_to_obj(filename)
    info = read_info(filename, verbose=false)

    if info == 1
        error("Glimpse only works for snapshots with info block, sorry!")
    end

	if nfiles == 1
	    # read blocks of particle data
	    bin_quantity = read_block_by_name(filename, blockname,
	                                      info=info[getfield.(info, :block_name) .== blockname][1],
	          					  		  parttype=0)

	    x = read_block_by_name(filename, "POS",
				         	   info=info[getfield.(info, :block_name) .== "POS"][1],
					   		   parttype=0)

	    rho = read_block_by_name(filename, "RHO",
	    				   info=info[getfield.(info, :block_name) .== "RHO"][1],
	    				   parttype=0)

	    hsml = read_block_by_name(filename, "HSML",
	    				      info=info[getfield.(info, :block_name) .== "HSML"][1],
	    					parttype=0)

	    m = read_block_by_name(filename, "MASS", parttype=0)
	else

		if center_pos == [123456.7, 123456.7, 123456.7]
			error("glimpse for multiple files works only with a given central position!")
		end

		if dx == 0.0 || dy == 0.0 || dz == 0.0
			error("glimpse for multiple files works only with a given extent!")
		end

		blocks = [blockname, "POS", "RHO", "HSML", "MASS"]
		unique!(blocks)

		x0 = center_pos - 0.5 .* [dx, dy, dz]
		x1 = center_pos + 0.5 .* [dx, dy, dz]

		d = read_particles_in_box(filebase, blocks, x0, x1, verbose=verbose)

		bin_quantity = d[blockname]
		x 	 = d["POS"]
		rho  = d["RHO"]
		hsml = d["HSML"]
		m 	 = d["MASS"]

		# "deallocate" d
		d = nothing
	end

    if kernel_name == "WC6"
        kernel = WendlandC6()
    elseif kernel_name == "WC4"
	  kernel = WendlandC4()
    elseif kernel_name == "Quntic"
	  kernel = Quintic()
    elseif kernel_name == "Cubic"
	  kernel = Cubic()
    end

    if center_pos == [123456.7, 123456.7, 123456.7]
		if verbose
        	@info "Calculating COM..."
    	end
        center_pos = calculate_center_of_mass(x, m)
		println("COM: $center_pos")
    end

    if [dx, dy, dz] == [0.0, 0.0, 0.0]
        dx = h.boxsize
	  	dy = h.boxsize
	  	dz = h.boxsize
    end

    max_size = maximum([dx, dy, dz])

    if run_dummy
	    par = mappingParameters(center = center_pos,
								x_size = dx, y_size = dy, z_size = dz,
	    				    	Npixels = 2)

	    if verbose
	    	@info "Initial compilation run..."
	    end
	    d = sphMapping(x, hsml, m, rho, bin_quantity,
						  param=par, kernel=kernel,
						  conserve_quantities=conserve_quantities,
						  parallel = parallel,
						  show_progress=false)
	end

    par = mappingParameters(center = center_pos,
							x_size = dx, y_size = dy, z_size = dz,
							Npixels = resolution)

    if verbose
		@info "Mapping..."
    end
    d = sphMapping(x, hsml, m, rho, bin_quantity,
					  param=par, kernel=kernel,
					  conserve_quantities=conserve_quantities,
					  parallel = parallel,
					  show_progress=verbose)

    if plot
		GR.imshow(d)
    end

    return d

end

"""
    sphMapping(Pos, HSML, M, ρ, Bin_Quant;
	           param::mappingParameters,
			   kernel::SPHKernel[,
			   show_progress::Bool=true,
			   conserve_quantities::Bool=true,
			   parallel::Bool=true,
			   dimensions::Int=2])

Maps the data in `Bin_Quant` to a grid. Parameters of mapping are supplied in
`param` and the kernel to be used in `kernel`.

# Arguments
- `Pos`: Array with particle positions.
- `HSML`: Array with particle hsml.
- `M`: Array with particle masses.
- `ρ`: Array with particle densities.
- `Bin_Quant`: Array with particle quantity to be mapped.
- `kernel::SPHKernel`: Kernel object to be used.
- `show_progress::Bool=true`: Show progress bar.
- `parallel::Bool=true`: Run on multiple processors.
- `conserve_quantities::Bool=true`: If quantities should be conserved while mapping, like in Smac (Dolag et. al. 2005).
- `dimensions::Int=2`: Number of mapping dimensions (2 = to grid, 3 = to cube).
"""
function sphMapping(Pos, HSML, M, ρ, Bin_Quant;
                    param::mappingParameters,
					kernel::SPHKernel,
                    show_progress::Bool=true,
					conserve_quantities::Bool=true,
					parallel::Bool=true,
					dimensions::Int=2)

	if (dimensions == 2)

		if !parallel
		 	return sphMapping_2D(Pos, HSML, M, ρ, Bin_Quant;
		 	                     param=param, kernel=kernel,
								 conserve_quantities=conserve_quantities,
		 	                     show_progress=show_progress)
		else
			@info "Running on $(nworkers()) cores."

			N = length(M)
			futures = Array{Future}(undef, nworkers())

			# 'Domain decomposition':
			# calculate array slices for each worker
			batch = Array{typeof(1:2)}(undef, nworkers())
			size = Int(floor(N/nworkers()))
			@inbounds for i = 1:nworkers()-1
			    batch[i] = 1+(i-1)*size:i*size
			end
			batch[nworkers()] = 1+(nworkers()-1)*size:N

			# start remote processes
			for (i, id) in enumerate(workers())
				futures[i] = @spawnat id sphMapping_2D(Pos[batch[i],:], HSML[batch[i]],
													   M[batch[i]], ρ[batch[i]],
													   Bin_Quant[batch[i]];
										   			   param=param, kernel=kernel,
										   			   conserve_quantities=conserve_quantities,
										   			   show_progress=false)
			end

			# get and reduce results
			return sum(fetch.(futures))
		end

	elseif (dimensions == 3 )
		return sphMapping_3D(Pos, HSML, M, ρ, Bin_Quant;
		                     param=param, kernel=kernel,
		                     show_progress=show_progress)
	end

end