using Knet, Test, Base.Iterators, IterTools, Random # , LinearAlgebra, StatsBase

include("common/debug.jl")
include("common/vocab.jl")
include("common/textreader.jl")
#include("common/embed.jl")
#include("common/linear.jl")
include("common/mask.jl")
include("common/mtdata.jl")


function trmor_word_tokenizer(line)
    (a,b) = split(line)
    return split(a, "")
end

function trmor_tag_tokenizer(line)
    (a,b) = split(line)
    c = split(b, ['+','^'])
    w = popfirst!(c)
    w = split(w, "")
    return append!(w, c)
end

trmor_file = "trmor2018.filtered"

trmor_word_vocab = Vocab(trmor_file; tokenizer=trmor_word_tokenizer)
trmor_tag_vocab = Vocab(trmor_file; tokenizer=trmor_tag_tokenizer)

trmor_word_reader = TextReader(trmor_file, trmor_word_vocab)
trmor_tag_reader = TextReader(trmor_file, trmor_tag_vocab)

# buckets based on src length, so make words src for clean uniform word length batches
trmor_data = MTData(trmor_word_reader, trmor_tag_reader; bucketwidth=1)
