# These don't work.

using Test
using CodeGen
using LLVM

# configure_logging(min_level=:info)
# configure_logging(min_level=:debug)
Base.CoreLogging.global_logger(Base.CoreLogging.SimpleLogger(STDERR, Base.CoreLogging.Debug))

f(x) = string("a", x)
m = codegen(f, Tuple{Int})
verify(m)

# @cgtest sqrt(2.0)    # intrinsic issue with add_ptr

# m = codegen(Base.Math.paynehanek, Tuple{Float64})
# m = codegen(Base.Math.rem_pio2_kernel, Tuple{Float64})
# @jitrun(Base.Math.rem_pio2_kernel, 1.1)
# nothing

# ci = code_typed(string, Tuple{String, String})
# mt = first(methods(string, Tuple{String, String}))
# m = codegen(string, Tuple{String, String})
# nothing
# m = codegen(Base.print_to_string, Tuple{Float64})
# m = codegen(sin_fast, Tuple{Float64}) # tries to call libopenlibm
# m = codegen(rand, Tuple{}) #  ccall issue
# m = codegen(sin, Tuple{Float64}) # compiles; verification fails big-time

# # Results in several problems below:  codegen of this works now
# codegen(Base.throw_inexacterror, Tuple{Symbol, Type{Int}, Int})

# abstract type AbstractA end
# struct A <: AbstractA
#     xx::Int
#     yy::Float64
# end
# testf() = 99

# cg = CodeGen.CodeCtx_init(testf, Tuple{})
# addr = LLVM.alloca!(cg.builder, CodeGen.llvmtype(A))
# gaddr = LLVM.struct_gep!(cg.builder, addr, 0)
# LLVM.store!(cg.builder, codegen!(cg, 3), gaddr)
# gaddr2 = LLVM.struct_gep!(cg.builder, addr, 1)
# LLVM.store!(cg.builder, codegen!(cg, 2.2), gaddr2)
# v = LLVM.load!(cg.builder, gaddr)
# LLVM.ret!(cg.builder, v)
# verify(cg.mod)

# array_max2(x) = maximum([3,x])
# m = codegen(array_max2, Tuple{Int})   # This runs; several verification problems
# # @cgtest array_max2(1)               # Segfaults...
# codegen(array_max2, Tuple{Float64})
# @jitrun(array_max2, 1.1)

# make_string(x) = string(1, x, "asdf")
# m = codegen(make_string, Tuple{Int}) # Unsupported intrinsic: arraylen

# function type_unstable(x)
#     for i = 1:10
#       x = x/2
#     end
#     return x
# end
# m = codegen(type_unstable, Tuple{Int})
# verify(m)
# nothing
# @jitrun(type_unstable, 1)   # segfaults

# # Wishful thinking:)
# using StaticArrays, OrdinaryDiffEq
# function diffeq()
#     A  = @SMatrix [ 1.0  0.0 0.0 -5.0
#                     4.0 -2.0 4.0 -3.0
#                    -4.0  0.0 0.0  1.0
#                     5.0 -2.0 2.0  3.0]
#     u0 = @SMatrix rand(4,2)
#     tspan = (0.0,1.0)
#     f(t,u) = A*u
#     prob = ODEProblem(f,u0,tspan)
#     solve(prob,Tsit5())
#     return 1
# end
# codegen(diffeq)


# Full Training Test (needs GPU for reasonable training time)using MuZero
using MuZero
import Zygote
using Grids
include("game.jl")
include("inference_tests.jl")
using Base.Threads: @spawn, @threads
config = getconfig(Tac)
config.batch_size = 16
config.root_dirichlet_alpha=0.3
config.num_actors = 3
config.weight_decay = 1e-4
config.lr_init=0.002f0
config.save_interval = 10000
config.num_simulations = 100
config.num_unroll_steps = 1
losses, dynamics_stats = [], []

policy_perfs = [];
p_tests = getpolicytests(Tac)
p_tests.period = 1000
testfn = gettestfn(p_tests)
function calc_policy_perf(l)
    acc, _ = testfn(l.test)
    push!(policy_perfs, acc)
end

test_cb = LearnerCallback(
    "test_cb",
    calc_policy_perf,
    period=1000,
)

learner = Learner(
    Tac;
    config=config,
    dir="output",
    cb=[test_cb],
    tests=[p_tests]
)

#selfplay!(learner; limit=config.batch_size)
# @spawn run_selfplay!(config, storage, replay_buffer; limit=33)
# @spawn run_selfplay!(config, storage, replay_buffer; limit=33)
# @spawn run_selfplay!(config, storage, replay_buffer; limit=34)
# rand(1:length(replay_buffer))

for i = 1:(config.num_actors)
    @spawn selfplay!(learner)
end
learner.replay_buffer |> length

#cb = [(1000, 1, [calc_policy_perf]),]

#config.training_steps = 3
#train_network!(learner, losses=losses, callbacks=cb)

config.training_steps = 200001
@show @time fit!(learner; steps=3)
@spawn fit!(learner)
#@spawn train_network!(learner, losses=losses, callbacks=cb)
#@spawn train_network!(config, storage, replay_buffer, losses=losses, callbacks=cb)

@show 1
learner.stats.losses |> length
@show 2
getpct.(policy_perfs)

using MuZero
using Base.Threads: @spawn

include("game.jl")
include("inference_tests.jl")
cd(@__DIR__)
config = getconfig(Tac)
config.checkpoint_interval = 1
config.save_interval = 1000
config.batch_size = 10
config.num_filters = 3
config.representation_resnet_layers = 1
config.dynamics_resnet_layers = 1
config.reward_fc_layers = 1
config.policy_fc_layers = 1
config.value_fc_layers = 1

pv_tests = get_policyvalue_tests(Tac)
pv_tests.period = 1
test_cb = LearnerCallback(
    "test_cb",
    (l)->println("hello from test_cb")
)

learner = Learner(
    Tac;
    dir="output",
    config=config,
    tests=[pv_tests],
    cb=[test_cb]
)

@spawn selfplay!(learner)
length(learner.replay_buffer)
fit!(learner, steps=10)

learner.stats.losses |> length
learner |> gettraindfall |> display
learner |> gettraindfall |> mean100 |> display
_, _, mse = gettestfn(pv_tests)(training_network(learner)) ; mse
_, _, mse = gettestfn(pv_tests)(test_network(learner)) ; mse

learner |> gettraindf |> display
learner |> gettraindfall |> mean100 |> display

using MuZero
include("game.jl")

show_policy_tests(learner)
policy_tests(learner)[3246] |> svg

g = Tac()
g |> display

step!(g, 4)
g |> display

legal_actions(g)

step!(g, 3)
g |> svg

step!(g, [7,9])
g |> svg

s = g |> get_state

isvalid(s)

ps = piece.(Tac, [:X, :O, :X, :O, :X, :O, :X, :O, :X])
s = TacState(ps, PLAYER_2)

ps2 = [piece.(Tac, [:X, :O, :X, :O, :X, :O, :X, :O])..., nothing]
s2 = TacState(ps2, PLAYER_2)

sp = getstatesplus(Tac) ; length(sp)
all_states = states(Tac) ; length(all_states)

getpolicytests(Tac)

target_s = "N0pMDQQAAAA0EAEIVGFjU3RhdGUfTgEETWFpbkQ0EAEKQm9hcmRTdGF0ZR8LnWwJrspjS75KQULJfWQ93gEGTXVaZXJvRAEAAADoFAkDAQMAAwIDAAMAAwADAAMAAwDg"
target_s |> decode |> display

s_plus = getstatesplus(Tac)

for i in 11:82
    @show i
    s_plus[i] |> display
end


#agent declaration
approximator_GNN = GeometricFlux.GraphConv(64 => 64, Flux.leakyrelu)
target_approximator_GNN = GeometricFlux.GraphConv(64 => 64, Flux.leakyrelu)
gnnlayers = 10
numInFeatures = 3
approximator_model = SeaPearl.CPNN(
    graphChain = Flux.Chain(
        GeometricFlux.GraphConv(numInFeatures => 64, Flux.leakyrelu),
        [approximator_GNN for i = 1:gnnlayers]...
    ),
    nodeChain = Flux.Chain(
        Flux.Dense(64, 32, Flux.leakyrelu),
        Flux.Dense(32, 32, Flux.leakyrelu),
        Flux.Dense(32, 16, Flux.leakyrelu),
    ),
    outputChain = Flux.Dense(16, 3),
)
target_approximator_model = SeaPearl.CPNN(
    graphChain = Flux.Chain(
        GeometricFlux.GraphConv(numInFeatures => 64, Flux.leakyrelu),
        [target_approximator_GNN for i = 1:gnnlayers]...
    ),
    nodeChain = Flux.Chain(
        Flux.Dense(64, 32, Flux.leakyrelu),
        Flux.Dense(32, 32, Flux.leakyrelu),
        Flux.Dense(32, 16, Flux.leakyrelu),
    ),
    outputChain = Flux.Dense(16, 3),
)

agent = RL.Agent(
    policy = RL.QBasedPolicy(
        learner = RL.DQNLearner(
            approximator = RL.NeuralNetworkApproximator(
                model = approximator_model,
                optimizer = ADAM(0.0005f0)
            ),
            target_approximator = RL.NeuralNetworkApproximator(
                model = target_approximator_model,
                optimizer = ADAM(0.0005f0)
            ),
            loss_func = Flux.Losses.huber_loss,
            stack_size = nothing,
            γ = 0.9999f0,
            batch_size = 1, #32,
            update_horizon = 25,
            min_replay_history = 1,
            update_freq = 10,
            target_update_freq = 200,
        ),
        explorer = RL.EpsilonGreedyExplorer(
            ϵ_stable = 0.001,
            kind = :exp,
            ϵ_init = 1.0,
            warmup_steps = 0,
            decay_steps = 5000,
            step = 1,
            is_break_tie = false,
            #is_training = true,
            rng = MersenneTwister(33)
        )
    ),
    trajectory = RL.CircularArraySARTTrajectory(
        capacity = 10,
        state = SeaPearl.DefaultTrajectoryState[] => (),
    )
)

@testset "basicmetrics.jl" begin

    @testset "constructor basicmetrics.jl" begin
        trailer = SeaPearl.Trailer()
        cpmodel = SeaPearl.CPModel(trailer)
        basicheuristic = SeaPearl.BasicHeuristic()

        metrics  = SeaPearl.BasicMetrics(cpmodel, basicheuristic)

        @test isempty(metrics.nodeVisited)== true
        @test isempty(metrics.meanNodeVisitedUntilEnd) == true
        @test isempty(metrics.timeneeded) == true
        @test isnothing(metrics.scores) == true
        @test isnothing(metrics.totalReward) == true
        @test isnothing(metrics.loss) == true
        @test metrics.nbEpisodes == 0

        metrics  = SeaPearl.BasicMetrics(cpmodel, basicheuristic;meanOver=100)
        @test metrics.meanOver == 100

        trailer = SeaPearl.Trailer()
        cpmodel = SeaPearl.CPModel(trailer)
        x = SeaPearl.IntVar(1, 2, "x", trailer)
        y = SeaPearl.IntVar(1, 2, "y", trailer)

        SeaPearl.addVariable!(cpmodel, x)
        SeaPearl.addVariable!(cpmodel, y)

        basicheuristic = SeaPearl.BasicHeuristic()
        learnedheuristic = SeaPearl.LearnedHeuristic(agent)

        metrics  = SeaPearl.BasicMetrics(cpmodel, basicheuristic)
        @test typeof(metrics) ==  SeaPearl.BasicMetrics{SeaPearl.DontTakeObjective, SeaPearl.BasicHeuristic}

        metrics  = SeaPearl.BasicMetrics(cpmodel, learnedheuristic)
        @test typeof(metrics) ==  SeaPearl.BasicMetrics{SeaPearl.DontTakeObjective, SeaPearl.LearnedHeuristic{SeaPearl.DefaultStateRepresentation{SeaPearl.DefaultFeaturization,SeaPearl.DefaultTrajectoryState}, SeaPearl.DefaultReward, SeaPearl.FixedOutput}}
        @test isnothing(metrics.totalReward) == false
        @test isnothing(metrics.loss) == false

        SeaPearl.addObjective!(cpmodel,y)

        metrics  = SeaPearl.BasicMetrics(cpmodel, basicheuristic)
        @test typeof(metrics) ==  SeaPearl.BasicMetrics{SeaPearl.TakeObjective, SeaPearl.BasicHeuristic}
        @test isnothing(metrics.scores) == false

        metrics  = SeaPearl.BasicMetrics(cpmodel, learnedheuristic)
        @test typeof(metrics) ==  SeaPearl.BasicMetrics{SeaPearl.TakeObjective, SeaPearl.LearnedHeuristic{SeaPearl.DefaultStateRepresentation{SeaPearl.DefaultFeaturization,SeaPearl.DefaultTrajectoryState}, SeaPearl.DefaultReward, SeaPearl.FixedOutput}}

    end

    @testset "BasicMetrics{DontTakeObjective, BasicHeuristic} " begin

        trailer = SeaPearl.Trailer()
        model = SeaPearl.CPModel(trailer)
        basicheuristic = SeaPearl.BasicHeuristic()

        x = SeaPearl.IntVar(2, 3, "x", trailer)
        y = SeaPearl.IntVar(2, 3, "y", trailer)
        SeaPearl.addVariable!(model, x)
        SeaPearl.addVariable!(model, y)
        SeaPearl.addConstraint!(model, SeaPearl.Equal(x, y, trailer))

        metrics  = SeaPearl.BasicMetrics(model, basicheuristic)
        dt = @elapsed SeaPearl.search!(model, SeaPearl.DFSearch(), SeaPearl.MinDomainVariableSelection(), basicheuristic)
        metrics(model,dt)

        @test metrics.nodeVisited[1] == [2,3]
        @test size(metrics.timeneeded,1) == 1
        @test metrics.nbEpisodes == 1

    end

    @testset "BasicMetrics{DontTakeObjective, LearnedHeuristic}" begin

        trailer = SeaPearl.Trailer()
        model = SeaPearl.CPModel(trailer)
        learnedheuristic = SeaPearl.LearnedHeuristic(agent)
        x = SeaPearl.IntVar(2, 3, "x", trailer)
        y = SeaPearl.IntVar(2, 3, "y", trailer)
        SeaPearl.addVariable!(model, x)
        SeaPearl.addVariable!(model, y)
        SeaPearl.addConstraint!(model, SeaPearl.Equal(x, y, trailer))

        metrics  = SeaPearl.BasicMetrics(model, learnedheuristic)
        dt = @elapsed SeaPearl.search!(model, SeaPearl.DFSearch(), SeaPearl.MinDomainVariableSelection(), learnedheuristic)

        metrics(model,dt)

        @test isempty(metrics.nodeVisited) == false #the number of solution found and their relative score depend on the stochastic heuristic...
        @test metrics.nbEpisodes == 1
        @test size(metrics.timeneeded,1) == 1
        @test size(metrics.totalReward,1) == 1
        @test size(metrics.loss,1) == 1

    end

    @testset "BasicMetrics{TakeObjective, BasicHeuristic}" begin
        trailer = SeaPearl.Trailer()
        model = SeaPearl.CPModel(trailer)
        basicheuristic = SeaPearl.BasicHeuristic()

        x = SeaPearl.IntVar(2, 3, "x", trailer)
        y = SeaPearl.IntVar(2, 3, "y", trailer)
        SeaPearl.addVariable!(model, x)
        SeaPearl.addVariable!(model, y)
        SeaPearl.addConstraint!(model, SeaPearl.Equal(x, y, trailer))
        SeaPearl.addObjective!(model,y)
        metrics  = SeaPearl.BasicMetrics(model, basicheuristic)
        dt = @elapsed SeaPearl.search!(model, SeaPearl.DFSearch(), SeaPearl.MinDomainVariableSelection(), basicheuristic)
        metrics(model,dt)

        @test metrics.nodeVisited[1] == [2, 3] #only one soluion found due to Objective prunning
        @test metrics.scores[1] == [3, 2]
        @test size(metrics.timeneeded,1) == 1
        @test metrics.nbEpisodes == 1
    end

    @testset "BasicMetrics{TakeObjective, LearnedHeuristic}" begin

        trailer = SeaPearl.Trailer()
        model = SeaPearl.CPModel(trailer)
        learnedheuristic = SeaPearl.LearnedHeuristic(agent)
        x = SeaPearl.IntVar(2, 3, "x", trailer)
        y = SeaPearl.IntVar(2, 3, "y", trailer)
        SeaPearl.addVariable!(model, x)
        SeaPearl.addVariable!(model, y)
        SeaPearl.addConstraint!(model, SeaPearl.Equal(x, y, trailer))
        SeaPearl.addObjective!(model,y)

        metrics  = SeaPearl.BasicMetrics(model, learnedheuristic)
        dt = @elapsed SeaPearl.search!(model, SeaPearl.DFSearch(), SeaPearl.MinDomainVariableSelection(), learnedheuristic)

        metrics(model,dt)

        @test isempty(metrics.nodeVisited) == false #the number of solution found and their relative score depend on the stochastic heuristic...
        @test isempty(metrics.scores) == false
        @test metrics.nbEpisodes == 1
        @test size(metrics.timeneeded,1) == 1
        @test size(metrics.totalReward,1) == 1
        @test size(metrics.loss,1) == 1
    end

    @testset "test !" begin
        trailer = SeaPearl.Trailer()
        model = SeaPearl.CPModel(trailer)
        basicheuristic = SeaPearl.BasicHeuristic()
        metrics  = SeaPearl.BasicMetrics(model, basicheuristic; meanOver=4)

        push!(metrics.nodeVisited, [10,0])
        push!(metrics.nodeVisited, [10,0])
        push!(metrics.nodeVisited, [10,0])
        push!(metrics.nodeVisited, [10,0])
        push!(metrics.nodeVisited, [0,10])
        push!(metrics.nodeVisited, [0,10])
        push!(metrics.nodeVisited, [0,10])
        push!(metrics.nodeVisited, [0,10])

        push!(metrics.meanNodeVisitedUntilEnd, 0)
        push!(metrics.meanNodeVisitedUntilEnd, 0)
        push!(metrics.meanNodeVisitedUntilEnd, 0)
        push!(metrics.meanNodeVisitedUntilEnd, 0)
        push!(metrics.meanNodeVisitedUntilEnd, 10)
        push!(metrics.meanNodeVisitedUntilEnd, 10)
        push!(metrics.meanNodeVisitedUntilEnd, 10)
        push!(metrics.meanNodeVisitedUntilEnd, 10)

        push!(metrics.meanNodeVisitedUntilfirstSolFound, 10)
        push!(metrics.meanNodeVisitedUntilfirstSolFound, 10)
        push!(metrics.meanNodeVisitedUntilfirstSolFound, 10)
        push!(metrics.meanNodeVisitedUntilfirstSolFound, 10)
        push!(metrics.meanNodeVisitedUntilfirstSolFound, 0)
        push!(metrics.meanNodeVisitedUntilfirstSolFound, 0)
        push!(metrics.meanNodeVisitedUntilfirstSolFound, 0)
        push!(metrics.meanNodeVisitedUntilfirstSolFound, 0)

        metrics.nbEpisodes = 8
        SeaPearl.computemean!(metrics)
        @test metrics.meanNodeVisitedUntilfirstSolFound == [10.0, 7.5, 5.0, 2.5, 0.0]
        @test metrics.meanNodeVisitedUntilEnd == [0.0, 2.5, 5.0, 7.5, 10.0]

    end
end


############################################################
# judiPDE ####################################################
############################################################

# Authors: Philipp Witte (pwitte@eos.ubc.ca), Henryk Modzelewski (hmodzelewski@eos.ubc.ca)
# Date: January 2017

export judiPDE, judiPDEexception, judiPDEadjoint, judiPDEadjointException, subsample

############################################################

# Type for linear operator representing  Pr*A(m)^-1*Ps,
# i.e. it includes source and receiver projections

# Forward
struct judiPDE{DDT<:Number,RDT<:Number} <: joAbstractLinearOperator{DDT,RDT}
    name::String
    m::Integer
    n::Integer
    info::Info
    model::Model
    geometry::Geometry
    options::Options
    fop::Function              # forward
    fop_T::Union{Function, Nothing}  # transpose
end

mutable struct judiPDEexception <: Exception
    msg :: String
end

# Adjoint
struct judiPDEadjoint{DDT<:Number,RDT<:Number} <: joAbstractLinearOperator{DDT,RDT}
    name::String
    m::Integer
    n::Integer
    info::Info
    model::Model
    geometry::Geometry
    options::Options
    fop::Function              # forward
    fop_T::Union{Function, Nothing}  # transpose
end

mutable struct judiPDEadjointException <: Exception
    msg :: String
end

############################################################
## outer constructors

function judiPDE(name::String,info::Info,model::Model, geometry::Geometry; options=Options(), DDT::DataType=Float32, RDT::DataType=DDT)
# JOLI wrapper for nonlinear forward modeling
    (DDT == Float32 && RDT == Float32) || throw(judiPDEexception("Domain and range types not supported"))
    if typeof(geometry) == GeometryOOC
        m = sum(geometry.nsamples)
    else
        m = 0
        for j=1:info.nsrc m += length(geometry.xloc[j])*geometry.nt[j] end
    end
    n = info.n * sum(info.nt)
    if info.nsrc > 1
        srcnum = 1:info.nsrc
    else
        srcnum = 1
    end
    return F = judiPDE{Float32,Float32}(name, m, n, info, model, geometry, options,
                              args -> time_modeling(model, args[1], args[2], args[3], args[4], args[5], srcnum, 'F', 1, options),
                              args_T -> time_modeling(model, args_T[1], args_T[2], args_T[3], args_T[4], args_T[5], srcnum, 'F', -1, options)
                              )
end

function judiPDEadjoint(name::String,info::Info,model::Model, geometry::Geometry; options=Options(), DDT::DataType=Float32, RDT::DataType=DDT)
# JOLI wrapper for nonlinear forward modeling
    (DDT == Float32 && RDT == Float32) || throw(judiPDEadjointException("Domain and range types not supported"))
    if typeof(geometry) == GeometryOOC
        m = sum(geometry.nsamples)
    else
        m = 0
        for j=1:info.nsrc m += length(geometry.xloc[j])*geometry.nt[j] end
    end
    n = info.n * sum(info.nt)
    if info.nsrc > 1
        srcnum = 1:info.nsrc
    else
        srcnum = 1
    end
    return F = judiPDEadjoint{Float32,Float32}(name, m, n, info, model, geometry, options,
                                args_T -> time_modeling(model, args_T[1], args_T[2], args_T[3], args_T[4], args_T[5], srcnum, 'F', -1, options),
                                args -> time_modeling(model, args[1], args[2], args[3], args[4], args[5], srcnum, 'F', 1, options),
                                )
end

############################################################
## overloaded Base functions

# conj(judiPDE)
conj(A::judiPDE{DDT,RDT}) where {DDT,RDT} =
    judiPDE{DDT,RDT}("conj("*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
        A.fop,
        A.fop_T
        )

# transpose(jo)
transpose(A::judiPDE{DDT,RDT}) where {DDT,RDT} =
    judiPDEadjoint{DDT,RDT}(A.name,A.n,A.m,A.info,A.model,A.geometry,A.options,
        A.fop_T,
        A.fop
        )

# adjoint(jo)
adjoint(A::judiPDE{DDT,RDT}) where {DDT,RDT} =
	judiPDEadjoint{DDT,RDT}(A.name,A.n,A.m,A.info,A.model,A.geometry,A.options,
		A.fop_T,
		A.fop
		)

# conj(jo)
conj(A::judiPDEadjoint{DDT,RDT}) where {DDT,RDT}=
    judiPDEadjoint{DDT,RDT}("conj("*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
        A.fop,
        A.fop_T
        )

# transpose(jo)
transpose(A::judiPDEadjoint{DDT,RDT}) where {DDT,RDT} =
    judiPDE{DDT,RDT}(A.name,A.n,A.m,A.info,A.model,A.geometry,A.options,
        A.fop_T,
        A.fop
        )
# adjoint(jo)
adjoint(A::judiPDEadjoint{DDT,RDT}) where {DDT,RDT} =
    judiPDE{DDT,RDT}(A.name,A.n,A.m,A.info,A.model,A.geometry,A.options,
        A.fop_T,
        A.fop
        )

############################################################
## overloaded Base *(...judi...)

# *(judiPDE,judiRHS)
function *(A::judiPDE{ADDT,ARDT},v::judiRHS{vDT}) where {ADDT,ARDT,vDT}
    A.n == size(v,1) || throw(judiPDEexception("shape mismatch"))
    jo_check_type_match(ADDT,vDT,join(["DDT for *(judiPDE,judiRHS):",A.name,typeof(A),vDT]," / "))
	args = (v.geometry,v.data,A.geometry,nothing,nothing)
    V = A.fop(args)
    jo_check_type_match(ARDT,eltype(V),join(["RDT from *(judiPDE,judiVector):",A.name,typeof(A),eltype(V)]," / "))
    return V
end

function *(A::judiPDEadjoint{ADDT,ARDT},v::judiRHS{vDT}) where {ADDT,ARDT,vDT}
    A.n == size(v,1) || throw(judiPDEadjointException("shape mismatch"))
    jo_check_type_match(ADDT,vDT,join(["DDT for *(judiPDE,judiRHS):",A.name,typeof(A),vDT]," / "))
	args = (A.geometry,nothing,v.geometry,v.data,nothing)
    V = A.fop(args)
    jo_check_type_match(ARDT,eltype(V),join(["RDT from *(judiPDE,judiVector):",A.name,typeof(A),eltype(V)]," / "))
    return V
end

# *(judiPDE,judiWavefield)
function *(A::judiPDE{ADDT,ARDT},v::judiWavefield{vDT}) where {ADDT,ARDT,vDT}
	A.n == size(v,1) || throw(judiPDEexception("shape mismatch"))
	jo_check_type_match(ADDT,vDT,join(["DDT for *(judiPDE,judiWavefield):",A.name,typeof(A),vDT]," / "))
	args = (nothing,v.data,A.geometry,nothing,nothing)
	V = A.fop(args)
	jo_check_type_match(ARDT,eltype(V),join(["RDT from *(judiPDE,judiVector):",A.name,typeof(A),eltype(V)]," / "))
	return V
end

function *(A::judiPDEadjoint{ADDT,ARDT},v::judiWavefield{vDT}) where {ADDT,ARDT,vDT}
	A.n == size(v,1) || throw(judiPDEadjointException("shape mismatch"))
	jo_check_type_match(ADDT,vDT,join(["DDT for *(judiPDE,judiWavefield):",A.name,typeof(A),vDT]," / "))
	args = (A.geometry,nothing,nothing,v.data,nothing)
	V = A.fop(args)
	jo_check_type_match(ARDT,eltype(V),join(["RDT from *(judiPDE,judiVector):",A.name,typeof(A),eltype(V)]," / "))
	return V
end

# *(judiPDE,judiProjection)
function *(A::judiPDE{CDT,ARDT},B::judiProjection{BDDT,CDT}) where {ARDT,BDDT,CDT}
    A.n == size(B,1) || throw(judiPDEexception("shape mismatch"))
    return judiModeling(A.info,A.model,B.geometry,A.geometry;options=A.options,DDT=CDT,RDT=ARDT)
end

function *(A::judiPDEadjoint{CDT,ARDT},B::judiProjection{BDDT,CDT}) where {ARDT,BDDT,CDT}
    A.n == size(B,1) || throw(judiPDEadjointException("shape mismatch"))
    return adjoint(judiModeling(A.info,A.model,A.geometry,B.geometry,options=A.options,DDT=CDT,RDT=ARDT))
end

# *(num,judiPDE)
function *(a::Number,A::judiPDE{ADDT,ARDT}) where {ADDT,ARDT}
    return judiPDE{ADDT,ARDT}("(N*"*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
        v1 -> jo_convert(ARDT,a*A.fop(v1),false),
        v2 -> jo_convert(ADDT,a*A.fop_T(v2),false)
        )
end

function *(a::Number,A::judiPDEadjoint{ADDT,ARDT}) where {ADDT,ARDT}
    return judiPDEadjoint{ADDT,ARDT}("(N*"*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
        v1 -> jo_convert(ARDT,a*A.fop(v1),false),
        v2 -> jo_convert(ADDT,a*A.fop_T(v2),false)
        )
end

############################################################
## overloaded Basees +(...judiPDE...), -(...judiPDE...)

# +(judiPDE,num)
function +(A::judiPDE{ADDT,ARDT},b::Number) where {ADDT,ARDT}
    return judiPDE{ADDT,ARDT}("("*A.name*"+N)",A.m,A.n,A.info,A.model,A.geometry,A.options,
        v1 -> A.fop(v1)+joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v1,
        v2 -> A.fop_T(v2)+joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v2
        )
end

function +(A::judiPDEadjoint{ADDT,ARDT},b::Number) where {ADDT,ARDT}
    return judiPDEadjoint{ADDT,ARDT}("("*A.name*"+N)",A.m,A.n,A.info,A.model,A.geometry,A.options,
        v1->A.fop(v1)+joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v1,
        v2->A.fop_T(v2)+joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v2
        )
end

# -(judiPDE,num)
function -(A::judiPDE{ADDT,ARDT},b::Number) where {ADDT,ARDT}
    return judiPDE{ADDT,ARDT}("("*A.name*"-N)",A.m,A.n,A.info,A.model,A.geometry,A.options,
        v1 -> A.fop(v1)-joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v1,
        v2 -> A.fop_T(v2)-joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v2
        )
end

function -(A::judiPDEadjoint{ADDT,ARDT},b::Number) where {ADDT,ARDT}
    return judiPDEadjoint{ADDT,ARDT}("("*A.name*"-N)",A.m,A.n,A.info,A.model,A.geometry,A.options,
        v1->A.fop(v1)-joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v1,
        v2->A.fop_T(v2)-joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v2
        )
end

# -(judiPDE)
-(A::judiPDE{DDT,RDT}) where {DDT,RDT} =
    judiPDE{DDT,RDT}("(-"*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
        v1->-A.fop(v1),
        v2->-A.fop_T(v2)
        )

-(A::judiPDEadjoint{DDT,RDT}) where {DDT,RDT} =
    judiPDEadjoint{DDT,RDT}("(-"*A.name*")",A.m,A.n,A.info,A.model,A.geometry,A.options,
        v1->-A.fop(v1),
        v2->-A.fop_T(v2)
        )

############################################################
## Additional overloaded functions

# Subsample Modeling operator
function subsample(F::judiPDE{ADDT,ARDT}, srcnum) where {ADDT,ARDT}
    geometry = subsample(F.geometry,srcnum)     # Geometry of subsampled data container
    info = Info(F.info.n, length(srcnum), F.info.nt[srcnum])
    return judiPDE(F.name, info, F.model, geometry; options=F.options, DDT=ADDT, RDT=ARDT)
end

function subsample(F::judiPDEadjoint{ADDT,ARDT}, srcnum) where {ADDT,ARDT}
    geometry = subsample(F.geometry,srcnum)     # Geometry of subsampled data container
    info = Info(F.info.n, length(srcnum), F.info.nt[srcnum])
    return judiPDEadjoint(F.name, info, F.model, geometry; options=F.options, DDT=ADDT, RDT=ARDT)
end

getindex(F::judiPDE,a) = subsample(F,a)
getindex(F::judiPDEadjoint,a) = subsample(F,a)


function init_from_experiment(experiment, data, labels, res)
    debug(LOGGER, "[INIT] Start initialization of experiment.")
    pools = copy(experiment[:param][:initial_pools])
    split_strategy = experiment[:split_strategy]

    # model
    train_data, train_pools, _ = get_train(split_strategy, data, pools)
    debug(LOGGER, "[INIT] Initializing model '$(experiment[:model])' with $(format_observations(train_data)) observations.")
    model = SVDD.instantiate(eval(experiment[:model][:type]), train_data, train_pools, experiment[:model][:param])
    SVDD.initialize!(model, eval(experiment[:model][:init_strategy]))
    set_model_fitted!(res, model)
    SVDD.set_adjust_K!(model, experiment[:param][:adjust_K])
    solver = experiment[:param][:solver]
    debug(LOGGER, "[INIT] Model solver for this experiment is '$(typeof(solver))'.")

    # query strategy
    debug(LOGGER, "[INIT] Initializing QS '$(experiment[:query_strategy][:type])'.")
    query_data, _, _ = get_query(split_strategy, data, pools)
    qs = initialize_qs(eval(experiment[:query_strategy][:type]), model, query_data, experiment[:query_strategy][:param])

    debug(LOGGER, "[INIT] Initializing oracle.")
    if isa(experiment[:oracle], Oracle)
        oracle = experiment[:oracle]
    else
        oracle = initialize_oracle(eval(experiment[:oracle][:type]), data, labels, experiment[:oracle][:param])
    end

    info(LOGGER, "[INIT] Initialization done.")
    return (model, pools, solver, qs, split_strategy, oracle)
end

function check_active_learn_args(data, labels)
    MLLabelUtils.islabelenc(labels, LABEL_ENCODING) || throw(ArgumentError("Argument labels is in the wrong encoding."))
    return size(data, 2) == length(labels) || throw(ArgumentError("Number of observations ($(size(data,2))) does not equal number of labels $(length(labels))."))
end

function active_learn(experiment::Dict{Symbol, Any})
    data, labels = load_data(experiment[:data_file])
    return active_learn(experiment, data, labels)
end

function active_learn(experiment::Dict{Symbol, Any}, data::Array{T, 2}, labels::Vector{Symbol}) where T <: Real
    check_active_learn_args(data, labels)
    log_experiment_info(experiment)

    res = Result(experiment)
    set_worker_info!(res)
    set_data_stats!(res, data, experiment[:split_strategy])

    model, pools, solver, qs, split_strategy, oracle = try
        init_from_experiment(experiment, data, labels, res)
    catch e
        warn(LOGGER, e)
        if isa(e, KDEException)
            res.status[:exit_code] = :kde_error
        elseif isa(e, MissingLabelTypeException)
            res.status[:exit_code] = :missing_label_type
        else
            throw(e)
        end
        return res
    end

    train_data, train_pools, _ = get_train(split_strategy, data, pools)
    SVDD.set_data!(model, train_data)
    SVDD.set_pools!(model, MLLabelUtils.labelmap(train_pools))

    classify_precision = get(experiment[:param], :classify_precision, SVDD.OPT_PRECISION)
    debug(LOGGER, "Classify precision: $classify_precision")

    debug(LOGGER, "Start active learning cycle with $(experiment[:param][:num_al_iterations]) queries.")
    for i in 0:experiment[:param][:num_al_iterations]
        info(LOGGER, "Iteration $(i)")
        debug(LOGGER, "Memory consumption $(round(Int, Sys.free_memory() / 2^20)) MB / $(round(Int, Sys.total_memory() / 2^20)) MB")

        debug(LOGGER, "[FIT] Start fitting model on $(format_observations(train_data)) observations.")
        # Workaround: redirect solver output
        stdout_orig, stderr_orig = stdout, stderr
        redirect_stdout(); redirect_stderr()
        status, time_fit, mem_fit = @timed SVDD.fit!(model, solver)
        redirect_stdout(stdout_orig); redirect_stderr(stderr_orig)
        debug(LOGGER, "[FIT] Fitting done ($(time_fit) s, $(format_bytes(mem_fit))).")

        ValueHistories.@trace res.al_history i time_fit mem_fit
        if status === JuMP.MathOptInterface.ALMOST_LOCALLY_SOLVED ||
            status === JuMP.MathOptInterface.LOCALLY_SOLVED
            warn(LOGGER, "[FIT] Model only locally solved. Solver status: $status.")
        elseif status !== JuMP.MathOptInterface.OPTIMAL
            warn(LOGGER, "[FIT] Model not solved to optimality. Solver status: $status.")
            res.status[:exit_code] = :solver_error
            return res
        end

        test_data, _, test_indices = get_test(split_strategy, data, pools)
        debug(LOGGER, "[TEST] Testing by predicting $(format_observations(test_data)) observations.")
        predictions = SVDD.predict(model, test_data)
        push_evaluation!(res.al_history, i, predictions, labels[test_indices], classify_precision)
        debug(LOGGER, "[TEST] Testing done.")

        if i < experiment[:param][:num_al_iterations]
            query_data, query_pools, query_indices = try
                get_query(split_strategy, data, pools)
            catch e
                warn(LOGGER, e)
                if isa(e, KDEException)
                     res.status[:exit_code] = :kde_error
                elseif isa(e, MissingLabelTypeException)
                    res.status[:exit_code] = :missing_label_type
                else
                    res.status[:exit_code] = :unknown_qs_error
                end
                return res
            end
            if :U ∉ query_pools && !isa(qs, QuerySynthesisStrategy)
                info(LOGGER, "Aborting '$(experiment[:hash])' after $(i) iterations because no more unlabeled observations are left.")
                al_summarize!(res)
                res.status[:exit_code] = :early_abort
                return res
            end
            debug(LOGGER, "[QS] Starting query strategy on $(format_observations(query_data)) observations.")
            if i == 0
                queries, time_qs, mem_qs = @timed get_query_objects_helper(qs, query_data, query_pools, query_indices)
            else
                queries, time_qs, mem_qs = @timed get_query_objects_helper(qs, query_data, query_pools, query_indices, values(res.al_history, :query_history))
            end

            debug(LOGGER, "[QS] Query strategy scoring done ($(time_qs) s, $(format_bytes(mem_qs))).")
            debug(LOGGER, "[QS] Updating pools.")
            query_labels = Oracles.ask_oracle(oracle, queries)
            data, pools, labels = process_queries!(queries, query_labels, model, split_strategy, data, pools, labels)
            debug(LOGGER, "[QS] Storing queries.")
            push_queries!(res.al_history, i, queries, query_labels, time_qs, mem_qs)
            isa(queries[1], Int) ? debug(LOGGER, "[QS] Query(ids = $(queries), labels = $(query_labels))") :
                                   debug(LOGGER, "[QS] Query(labels = $(query_labels))")
            push_eval_queries!(res.al_history, i, model, data, queries)
            debug(LOGGER, "[QS] Query strategy done.")
        end
        debug(LOGGER, "Finished iteration $(i).")
    end
    debug(LOGGER, "Finished active learning cycle.")

    al_summarize!(res)
    debug(LOGGER, "Summary done.")
    info(LOGGER, "Finished experiment '$(experiment[:hash])'.")
    res.status[:exit_code] = :success
    return res
end

function process_queries!(query_data::Array{T, 2},
                          query_labels::Vector{Symbol},
                          model, split_strategy, data, pools, labels) where T <: Real
    size(query_data, 2) == length(query_labels) || throw(DimensionMismatch("Number of queries does not match number of labels."))
    size(data, 1) == size(query_data, 1) || throw(DimensionMismatch("Data dimensionality does not match query dimensionality."))

    n_old = size(data,2)
    append!(split_strategy.train, trues(length(query_labels)))
    append!(split_strategy.test, falses(length(query_labels)))
    data = hcat(data, query_data)
    pools = vcat(pools, fill(:U, length(query_labels)))
    labels = vcat(labels, query_labels)
    n_new = size(data, 2)
    global_query_ids = collect((n_old + 1):n_new)

    process_queries!(global_query_ids, query_labels, model, split_strategy, data, pools, labels)
end

function process_queries!(global_query_ids::Vector{Int},
                          query_labels::Vector{Symbol},
                          model, split_strategy, data, pools, labels)
    length(global_query_ids) == length(query_labels) || throw(DimensionMismatch("Number of queries does not match number of labels."))

    pools_before = copy(pools)
    train_mask_before = calc_mask(split_strategy.train_strat, split_strategy.train, pools)

    query_pool_labels = convert_labels_to_learning(query_labels)
    pools[global_query_ids] .= query_pool_labels
    train_data, train_pools, _ = get_train(split_strategy, data, pools)

    train_mask_after = calc_mask(split_strategy.train_strat, split_strategy.train, pools)

    # indices that are in train before and after, relative to the updated train
    global_remaining_indices = findall(train_mask_before .& train_mask_after)
    old_idx_remaining_train = map(id -> get_local_idx(id, split_strategy, pools_before, Val(:train)), global_remaining_indices)
    new_idx_remaining_train = map(id -> get_local_idx(id, split_strategy, pools, Val(:train)), global_remaining_indices)

    filtered_query_ids = filter_query_id(global_query_ids, split_strategy, query_pool_labels, Val(:train))
    # check if query is relevant for train subset and model
    if isempty(filtered_query_ids)
        return data, pools, labels
    end
    train_query_ids = map(id -> get_local_idx(id, split_strategy, pools, Val(:train)), filtered_query_ids)

    SVDD.update_with_feedback!(model, train_data, train_pools, train_query_ids, old_idx_remaining_train, new_idx_remaining_train)

    return data, pools, labels
end

function get_query_objects_helper(qs::Q,
                                 query_data::Array{T, 2},
                                 query_pools::Vector{Symbol},
                                 query_indices::Vector{Int},
                                 history::Vector{Vector{Int}}=Vector{Vector{Int}}())::Vector{Int} where T <: Real where Q <: Union{PoolQs, SubspaceQs}
    return get_query_objects(qs, query_data, query_pools, query_indices, history)
end

function get_query_objects_helper(qs::QuerySynthesisStrategy,
                                 query_data::Array{T, 2},
                                 query_pools::Vector{Symbol},
                                 query_indices::Vector{Int},
                                 history::Vector{Array{T, 2}}=Vector{Array{T, 2}}())::Array{T, 2} where T <: Real
    return get_query_objects(qs, query_data, query_pools, history)
end

function push_queries!(al_history::ValueHistories.MVHistory, i, queries, query_labels, time_qs, mem_qs)
    push!(al_history, :query_history, i, queries)
    ValueHistories.@trace al_history i query_labels time_qs mem_qs
    return nothing
end

function push_eval_queries!(al_history::ValueHistories.MVHistory, i, model::OCClassifier, data::Array{T, 2}, queries::Vector{Int}) where T <: Real
    return push_eval_queries!(al_history, i, model, data, data[:, queries])
end

function push_eval_queries!(al_history::ValueHistories.MVHistory, i, model::OCClassifier, data::Array{T, 2}, queries::Array{T, 2}) where T <: Real
    if size(queries, 2) < 2
        angle_div, euclidean_div = 0.0, 0.0
    else
        kernel = SVDD.get_kernel(model)
        angle_div = kernel !== nothing ? angle_batch_diversity(kernel, queries) : -1
        euclidean_div = euclidean_batch_diversity(queries)
    end
    push!(al_history, :angle_batch_diversity, i, angle_div)
    push!(al_history, :euclidean_batch_diversity, i, euclidean_div)
    return nothing
end

function push_evaluation_cm!(al_history, i, cm)
    push!(al_history, :cm, i, cm)
    for e in [:cohens_kappa, :matthews_corr, :f1_score, :tpr, :fpr]
        push!(al_history, e, i, eval(e)(cm))
    end
end

function push_evaluation!(al_history::ValueHistories.MVHistory, i, predictions::Vector{Vector{Float64}}, labels, classify_precision)
    cm = ConfusionMatrix(SVDD.classify(predictions, Val(:Global), opt_precision=classify_precision), labels)
    push_evaluation_cm!(al_history, i, cm)
    return nothing
end

function push_evaluation!(al_history::ValueHistories.MVHistory, i, predictions, labels, classify_precision)
    cm = ConfusionMatrix(SVDD.classify.(predictions, opt_precision=classify_precision), labels)
    push_evaluation_cm!(al_history, i, cm)
    push!(al_history, :auc, i, roc_auc(predictions, labels))
    for k in [0.01, 0.02, 0.05, 0.1, 0.2]
        auc_fpr = roc_auc(predictions, labels, fpr = k)
        auc_fpr_normalized = roc_auc(predictions, labels, fpr = k, normalize = true)
        push!(al_history, Symbol("auc_fpr_$(k)"), i, auc_fpr)
        push!(al_history, Symbol("auc_fpr_normalized_$(k)"), i, auc_fpr_normalized)
    end
    return nothing
end


using Match

const PulseTable = map(x -> 95.52f0 / (8128.0f0/Float32(x) + 100.0f0), 0:30)

const TNDTable = map(x -> 163.67f0 / (24329.0f0/Float32(x) + 100.0f0), 0:202)

function apustep!(console::Console, apu::APU)
  apu.cycle += 1
  steptimer!(console, apu)
  #f1 = floor(Int32, Float64(cycle1) / frameCounterRate)
  #f2 = floor(Int32, Float64(cycle2) / frameCounterRate)
  apu.frameCounter += 240
  if apu.frameCounter >= cpuFrequency
    stepframecounter!(console)
    apu.frameCounter -= cpuFrequency
  end
  apu.sampleCounter += 1.0
  if apu.sampleCounter >= apu.sampleRate
    sendsample!(apu)
    apu.sampleCounter -= apu.sampleRate
  end
  #s1 = floor(Int32, Float64(cycle1) / apu.sampleRate)
  #s2 = floor(Int32, Float64(cycle2) / apu.sampleRate)
  #if s1 != s2
    #sendsample!(apu)
  #end
end

# mode 0:    mode 1:       function
# ---------  -----------  -----------------------------
#  - - - f    - - - - -    IRQ (if bit 6 is clear)
#  - l - l    l - l - -    Length counter and sweep
#  e e e e    e e e e -    Envelope and linear counter
function stepframecounter!(console::Console)
  apu = console.apu
  @match apu.framePeriod begin
    0x04 => begin
      apu.frameValue = (apu.frameValue + 0x01) & 0x03
      @match apu.frameValue begin
        0x00 => begin
          stepenvelope!(apu)
        end
        0x01 => begin
          stepenvelope!(apu)
          stepsweep!(apu)
          steplength!(apu)
        end
        0x02 => begin
          stepenvelope!(apu)
        end
        0x03 => begin
          stepenvelope!(apu)
          stepsweep!(apu)
          steplength!(apu)
          fireirq!(console)
        end
      end
    end
    0x05 => begin
      apu.frameValue = (apu.frameValue + 0x01) % 0x05
      @match apu.frameValue begin
        0x00 => begin
          stepenvelope!(apu)
        end
        0x01 => begin
          stepenvelope!(apu)
          stepsweep!(apu)
          steplength!(apu)
        end
        0x02 => begin
          stepenvelope!(apu)
        end
        0x04 => begin
          stepenvelope!(apu)
          stepsweep!(apu)
          steplength!(apu)
        end
      end
    end
  end
end

function sendsample!(apu::APU)
  out = step!(apu.filterChain, output!(apu))
  write!(apu.channel, out)
end

function output!(apu::APU)::Float32
  p1 = output!(apu.pulse1)
  p2 = output!(apu.pulse2)
  t = output!(apu.triangle)
  n = output!(apu.noise)
  d = output!(apu.dmc)
  pulseOut = PulseTable[p1 + p2 + 1]
  tndOut = TNDTable[3 * t + 2 * n + d + 1]
  return pulseOut + tndOut
end

function stepenvelope!(apu::APU)
  stepenvelope!(apu.pulse1)
  stepenvelope!(apu.pulse2)
  stepcounter!(apu.triangle)
  stepenvelope!(apu.noise)
end

function stepsweep!(apu::APU)
  stepsweep!(apu.pulse1)
  stepsweep!(apu.pulse2)
end

function steplength!(apu::APU)
  steplength!(apu.pulse1)
  steplength!(apu.pulse2)
  steplength!(apu.triangle)
  steplength!(apu.noise)
end

function readregister(apu::APU, address::UInt16)::UInt8
  @match address begin
    0x4015 => readstatus(apu)
    _ => begin
      # default:
      # TODO: add logging.
      # log.Fatalf("unhandled apu register read at address: 0x%04X", address)
      0x00
    end
  end
end

function apuwriteregister!(console::Console, address::UInt16, val::UInt8)
  apu = console.apu
  @match address begin
    0x4000 => begin
      writecontrol!(apu.pulse1, val)
    end
    0x4001 => begin
      writesweep!(apu.pulse1, val)
    end
    0x4002 => begin
      writetimerlow!(apu.pulse1, val)
    end
    0x4003 => begin
      writetimerhigh!(apu.pulse1, val)
    end
    0x4004 => begin
      writecontrol!(apu.pulse2, val)
    end
    0x4005 => begin
      writesweep!(apu.pulse2, val)
    end
    0x4006 => begin
      writetimerlow!(apu.pulse2, val)
    end
    0x4007 => begin
      writetimerhigh!(apu.pulse2, val)
    end
    0x4008 => begin
      writecontrol!(apu.triangle, val)
    end
    0x4009 => begin
    end
    0x4010 => begin
      writecontrol!(apu.dmc, val)
    end
    0x4011 => begin
      writevalue!(apu.dmc, val)
    end
    0x4012 => begin
      writeaddress!(apu.dmc, val)
    end
    0x4013 => begin
      writelength!(apu.dmc, val)
    end
    0x400A => begin
      writetimerlow!(apu.triangle, val)
    end
    0x400B => begin
      writetimerhigh!(apu.triangle, val)
    end
    0x400C => begin
      writecontrol!(apu.noise, val)
    end
    0x400D => begin
    end
    0x400E => begin
      writeperiod!(apu.noise, val)
    end
    0x400F => begin
      writelength!(apu.noise, val)
    end
    0x4015 => begin
      writecontrol!(apu, val)
    end
    0x4017 => begin
      writeframecounter!(apu, val)
    end
    _ => begin
      # default:
      #   log.Fatalf("unhandled apu register write at address: 0x%04X", address)
    end
  end
end

function readstatus(apu::APU)::UInt8
  result = 0x00
  if apu.pulse1.lengthValue > 0
    result |= 0x01
  end
  if apu.pulse2.lengthValue > 0
    result |= 0x02
  end
  if apu.triangle.lengthValue > 0
    result |= 0x04
  end
  if apu.noise.lengthValue > 0
    result |= 0x08
  end
  if apu.dmc.currentLength > 0
    result |= 0x10
  end
  return result
end

function writecontrol!(apu::APU, val::UInt8)
  pulse1 = apu.pulse1
  pulse1.enabled = (val & 1) == 1
  if !pulse1.enabled
    pulse1.lengthValue = 0
  end
  pulse2 = apu.pulse2
  pulse2.enabled = (val & 2) == 2
  if !pulse2.enabled
    pulse2.lengthValue = 0
  end
  triangle = apu.triangle
  triangle.enabled = (val & 4) == 4
  if !triangle.enabled
    triangle.lengthValue = 0
  end
  noise = apu.noise
  noise.enabled = (val & 8) == 8
  if !noise.enabled
    noise.lengthValue = 0
  end
  dmc = apu.dmc
  dmc.enabled = (val & 16) == 16
  if !dmc.enabled
    dmc.currentLength = 0
  else
    if dmc.currentLength == 0
      restart!(dmc)
    end
  end
end

function writeframecounter!(apu::APU, val::UInt8)
  apu.framePeriod = 0x04 + ((val >> 7) & 0x01)
  apu.frameIRQ = ((val >> 6) & 0x01) == 0x00
  # frameValue = 0
  if apu.framePeriod == 0x05
    stepenvelope!(apu)
    stepsweep!(apu)
    steplength!(apu)
  end
end


const FRACTAL_HEAP_HEADER_SIGNATURE = htol(0x50485246) # UInt8['F','R','H','P']
const FRACTAL_HEAP_INDIRECT_BLOCK_SIGNATURE = htol(0x42494846) # UInt8['F','H','I','B']
const FRACTAL_HEAP_DIRECT_BLOCK_SIGNATURE = htol(0x42444846) # UInt8['F', 'H', 'D', 'B']

struct FractalHeapHeader
    offset::RelOffset
    table_width::Int
    starting_block_size::Int
    max_direct_block_size::Int
    max_heap_size::Int
    root_block_address::RelOffset
    cur_num_rows_in_root_iblock::Int
    has_io_filter::Bool
    max_dblock_rows::Int
    max_size_managed_objects::Int
    # could add the rest of the fields if they ever become necessary
end

struct FractalHeapDirectBlock
    offset::RelOffset # position of block in file
    # block offset in heaps address space
    # WARNING: don't use. sometimes wrong in long files
    block_offset::UInt64
    size::UInt64
    filtered_size::UInt64 # set to typemax if not filtered
    filter_mask::UInt32  # set to typemax if not filtered
end

struct FractalHeapIndirectBlock
    offset::RelOffset # position of iblock in file
    block_offset::UInt64 # block offset in heaps address space
    dblocks::Vector{FractalHeapDirectBlock}
    iblocks::Vector{FractalHeapIndirectBlock}
end

function blocksize(blocknum, starting_size, table_width)
    #block numbering starts at zero
    rownum = Int(blocknum ÷ table_width)
    (2^(max(0,rownum-1))) * starting_size
end

function block_num_size_start(offset, hh)
    width = hh.table_width
    # first compute row number
    r = Int(offset ÷ (hh.starting_block_size*width))
    r > 2 && (r = ceil(Int, log2(r+1)))
    # row start offset
    row_startoffset = (r>1 ? 2^(r-1) : r)*hh.starting_block_size*width
    block_size = (2^(max(0,r-1))) * hh.starting_block_size
    block_num = width*r + (offset-row_startoffset) ÷ block_size
    block_start = row_startoffset + block_size*(block_num-width*r)
    block_num, block_size, block_start
end

function read_fractal_heap_header(f, offset)
    OffsetType = uintofsize(f.superblock.size_of_offsets)
    LengthType = uintofsize(f.superblock.size_of_lengths)

    io = f.io
    seek(io, fileoffset(f, offset)) # may need to compute fileoffset
    cio = begin_checksum_read(io)

    signature = jlread(cio, UInt32)
    signature == FRACTAL_HEAP_HEADER_SIGNATURE || throw(InvalidDataException("Signature does not match."))

    version = jlread(cio, UInt8)
    heap_id_length = jlread(cio, UInt16)
    io_filter_encoded_length = jlread(cio, UInt16)
    flags = jlread(cio, UInt8)
    max_size_managed_objects = jlread(cio, UInt32)
    next_huge_object_id = jlread(cio, LengthType)
    huge_object_v2btree_address = jlread(cio, OffsetType)
    free_space_in_managed_blocks = jlread(cio, LengthType)
    managed_block_free_space_manager = jlread(cio, OffsetType)
    managed_space_in_heap = jlread(cio, LengthType)
    allocated_space_in_heap = jlread(cio, LengthType)
    direct_block_allocation_iterator_offset = jlread(cio, LengthType)
    managed_objects_number_in_heap = jlread(cio, LengthType)
    huge_objects_size_in_heap = jlread(cio, LengthType)
    huge_objects_number_in_heap = jlread(cio, LengthType)
    tiny_objects_size_in_heap = jlread(cio, LengthType)
    tiny_objects_number_in_heap = jlread(cio, LengthType)

    table_width = jlread(cio, UInt16)
    starting_block_size = jlread(cio, LengthType)
    max_direct_block_size = jlread(cio, LengthType)
    max_heap_size = jlread(cio, UInt16)
    num_starting_rows_in_root_iblock = jlread(cio, UInt16)
    root_block_address = jlread(cio, RelOffset)#OffsetType) # RelOffset ?
    cur_num_rows_in_root_iblock = jlread(cio, UInt16)

    has_io_filter = io_filter_encoded_length > 0
    if has_io_filter
        filtered_root_direct_block_size = jlread(cio, LengthType)
        io_filter_mask = jlread(cio, UInt32)
        io_filter_information = jlread(cio, UInt8, io_filter_encoded_length)
    else
        filtered_root_direct_block_size = typemax(LengthType)
        io_filter_mask = typemax(UInt32)
        io_filter_information = UInt8[]
    end

    # Checksum
    end_checksum(cio) == jlread(io, UInt32) || throw(InvalidDataException("Invalid Checksum"))

    max_dblock_rows = (log2(max_direct_block_size) - log2(starting_block_size))+2 |> Int

    FractalHeapHeader(offset, table_width, starting_block_size, max_direct_block_size, max_heap_size,
        root_block_address, cur_num_rows_in_root_iblock, has_io_filter, max_dblock_rows,
        max_size_managed_objects)
end

function read_indirect_block(f, offset, hh, nrows)
    OffsetType = uintofsize(f.superblock.size_of_offsets)
    LengthType = uintofsize(f.superblock.size_of_lengths)

    io = f.io
    seek(io, fileoffset(f, offset))
    cio = begin_checksum_read(io)

    signature = jlread(cio, UInt32)
    signature == FRACTAL_HEAP_INDIRECT_BLOCK_SIGNATURE || throw(InvalidDataException("Signature does not match."))

    version = jlread(cio, UInt8)
    heap_header_address = jlread(cio, OffsetType)
    # number of bytes for block offset
    offset_byte_num = ceil(Int, hh.max_heap_size / 8)
    block_offset = to_uint64(jlread(cio, UInt8, offset_byte_num))

    # Read child direct blocks
    block_start = block_offset
    K = min(nrows, hh.max_dblock_rows)*hh.table_width
    dblocks = map(1:K) do k
        dblock_address = jlread(cio, RelOffset) # OffsetType
        dblock_size = blocksize(k-1, hh.starting_block_size, hh.table_width)
        if hh.has_io_filter > 0
            filtered_size = jlread(cio, LengthType)
            filter_mask = jlread(cio, UInt32)
        else
            filtered_size = typemax(UInt64)
            filter_mask = typemax(UInt32)
        end
        dblock = FractalHeapDirectBlock(dblock_address, block_start, dblock_size, filtered_size, filter_mask)
        block_start += dblock_size
        return dblock
    end
    N = (nrows <= hh.max_dblock_rows) ? 0 :  (nrows-hh.max_dblock_rows)*hh.table_width
    iblock_addresses = map(1:N) do n
        jlread(cio, RelOffset) #OffsetType
    end

    # Checksum
    end_checksum(cio) == jlread(io, UInt32) || throw(InvalidDataException())

    iblocks = Vector{FractalHeapIndirectBlock}(undef, N)
    for n=1:N
        iblock_offset = iblock_addresses[n]
        iblock_offset == UNDEFINED_ADDRESS && break
        # figure out iblock size / nrows
        block_num = K+(n-1)
        rownum = block_num ÷ hh.table_width
        block_size = (2^(max(0,rownum-1))) * hh.starting_block_size
        sub_iblock_nrows = (log2(block_size)-log2(hh.starting_block_size* hh.table_width))+1
        iblocks[n] = read_indirect_block(f, iblock_offset , hh, sub_iblock_nrows)
    end
    FractalHeapIndirectBlock(offset, block_offset, dblocks, iblocks)
end

#####################################################################################################
## Version 2 B-trees
#####################################################################################################

const V2_BTREE_HEADER_SIGNATURE = htol(0x44485442) # UInt8['B','T','H','D']
const V2_BTREE_INTERNAL_NODE_SIGNATURE = htol(0x4e495442) # UInt8['B', 'T', 'I', 'N']
const V2_BTREE_LEAF_NODE_SIGNATURE = htol(0x464c5442) # UInt8['B', 'T', 'L', 'F']

struct BTreeHeaderV2
    offset::RelOffset
    type::Int
    node_size::Int
    record_size::Int
    depth::Int
    split_percent::Int
    merge_percent::Int
    root_node_address::RelOffset
    num_records_in_root_node::Int
    num_records_in_tree::Int
end

abstract type BTreeNodeV2 end
abstract type BTreeRecordV2 end

struct BTreeInternalNodeV2 <: BTreeNodeV2
    offset::RelOffset
    type::UInt8
    records::Vector{Any}
    child_nodes::Vector #abstract to defer loading
end

struct BTreeLeafNodeV2 <: BTreeNodeV2
    offset::RelOffset
    type::UInt8
    records::Vector{<:BTreeRecordV2}
end

struct BTreeType5RecordV2 <: BTreeRecordV2
    hash::UInt32
    offset::UInt64
    length::Int
end

function read_v2btree_header(f, offset)
    OffsetType = uintofsize(f.superblock.size_of_offsets)
    LengthType = uintofsize(f.superblock.size_of_lengths)

    io = f.io
    seek(io, fileoffset(f, offset))
    cio = begin_checksum_read(io)

    signature = jlread(cio, UInt32)
    signature == V2_BTREE_HEADER_SIGNATURE || throw(InvalidDataException("Signature does not match."))

    version = jlread(cio, UInt8)
    type = jlread(cio, UInt8)
    node_size = jlread(cio, UInt32)
    record_size = jlread(cio, UInt16)
    depth = jlread(cio, UInt16)
    split_percent = jlread(cio, UInt8)
    merge_percent = jlread(cio, UInt8)
    root_node_address = jlread(cio, RelOffset)#OffsetType)
    num_records_in_root_node = jlread(cio, UInt16)
    num_records_in_tree = jlread(cio, LengthType)

    end_checksum(cio) == jlread(io, UInt32) || throw(InvalidDataException())
    BTreeHeaderV2(  offset,
                    type,
                    node_size,
                    record_size,
                    depth,
                    split_percent,
                    merge_percent,
                    root_node_address,
                    num_records_in_root_node,
                    num_records_in_tree)
end

function read_v2btree_node(f, offset, num_records, depth, bh, hh)
    if depth == 0
        return read_v2btree_leaf_node(f, offset, num_records, bh, hh)
    end
    OffsetType = uintofsize(f.superblock.size_of_offsets)
    LengthType = uintofsize(f.superblock.size_of_lengths)

    io = f.io
    seek(io, fileoffset(f, offset)) # may need to compute fileoffset
    cio = begin_checksum_read(io)

    signature = jlread(cio, UInt32)
    signature == V2_BTREE_INTERNAL_NODE_SIGNATURE || throw(InvalidDataException("Signature does not match."))

    version = jlread(cio, UInt8)
    type = jlread(cio, UInt8)

    records = map(1:num_records) do n
        read_record(cio, type, hh)
    end

    # determine number of bytes used to encode `num_records`
    # this has to be done iteratively
    # leaf node:
    space = bh.node_size - 4 - 1 - 1 - 4
    max_records = space ÷ bh.record_size
    max_records_total = 0
    numbytes = size_size(max_records)
    numbytes_total = 0

    for d = 1:depth
        space = bh.node_size - 4-1-1-4 - sizeof(RelOffset) - (d>1)*numbytes_total
        max_records = space ÷ (bh.record_size + sizeof(RelOffset) + numbytes+(d>1)*numbytes_total)
        numbytes = size_size(max_records)
        max_records_total = max_records + (max_records+1)*max_records_total
        numbytes_total = size_size(max_records_total)
    end
    numbytes_total = size_size2(max_records_total)
    child_nodes = map(1:num_records+1) do _
        child_node_pointer = jlread(cio, RelOffset) # offset type
        num_records = Int(to_uint64(jlread(cio, UInt8, numbytes)))
        if depth > 1
            total_records = Int(to_uint64(jlread(cio, UInt8, numbytes_total)))
            return (; child_node_pointer, num_records,total_records)
        end
        (; child_node_pointer, num_records)
    end
    end_checksum(cio) == jlread(io, UInt32) || throw(InvalidDataException())

    BTreeInternalNodeV2(offset, type, records, child_nodes)
end

function read_v2btree_leaf_node(f, offset, num_records, bh, hh)
    OffsetType = uintofsize(f.superblock.size_of_offsets)
    LengthType = uintofsize(f.superblock.size_of_lengths)

    io = f.io
    seek(io, fileoffset(f, offset)) # may need to compute fileoffset
    cio = begin_checksum_read(io)

    signature = jlread(cio, UInt32)
    signature == V2_BTREE_LEAF_NODE_SIGNATURE || throw(InvalidDataException("Signature does not match."))
    version = jlread(cio, UInt8)
    type = jlread(cio, UInt8)
    records = map(1:num_records) do n
        read_record(cio, type, hh)
    end

    end_checksum(cio) == jlread(io, UInt32) || throw(InvalidDataException())
    BTreeLeafNodeV2(offset, type, records)
end

function read_record(io, type, hh)
    if type == 5 # link name for indexed group
        hash_of_name = jlread(io, UInt32)
        # read Heap id for managed object
        version_type = jlread(io, UInt8)

        offbytes = hh.max_heap_size÷8
        offset =Int(to_uint64(jlread(io, UInt8, offbytes)))
        lnbytes = min(hh.max_direct_block_size, hh.max_size_managed_objects) |> size_size2
        length = Int(to_uint64(jlread(io, UInt8, lnbytes)))
        skip(io, 6-offbytes-lnbytes)
        return BTreeType5RecordV2(hash_of_name, offset, length)
    else
        throw(error("Not implemented record type"))
    end
end

function read_records_in_node(f, offset, num_records, depth, bh, hh)
    if depth == 0
        return read_v2btree_leaf_node(f, offset, num_records, bh, hh).records
    end

    node = read_v2btree_node(f, offset, num_records, depth, bh, hh)

    records = []
    for n=1:num_records+1
        child_offset = node.child_nodes[n].child_node_pointer
        child_records = node.child_nodes[n].num_records
        records_in_child = read_records_in_node(f, child_offset, child_records, depth-1, bh, hh)
        append!(records, records_in_child)
        n<=num_records && (push!(records, node.records[n]))
    end
    return records
end

function get_block_offset(f, iblock, roffset, hh)
    block_num, block_size, block_start = block_num_size_start(roffset, hh)
    K = length(iblock.dblocks)
    if block_num < K
        dblock = iblock.dblocks[block_num+1]
        return fileoffset(f,dblock.offset) + roffset - block_start
    end
    sub_iblock =  iblock.iblocks[block_num-K+1]
    get_block_offset(f, sub_iblock, roffset-block_start, hh)
end

function read_btree(f, offset_hh, offset_bh)
    hh = read_fractal_heap_header(f, offset_hh)
    bh = read_v2btree_header(f, offset_bh)

    records = read_records_in_node(f, bh.root_node_address, bh.num_records_in_root_node, bh.depth, bh, hh)
    indirect_rb = read_indirect_block(f, hh.root_block_address, hh, hh.cur_num_rows_in_root_iblock)
    links = map(records) do r
        offset = get_block_offset(f, indirect_rb, r.offset, hh)
        seek(f.io, offset)
        read_link(f.io)
    end
    links
end


# This file is a part of MJDSigGen, licensed under the MIT License (MIT).

struct CartPoint{T<:Real}
    x::T
    y::T
    z::T
end

CartPoint{T}() where {T} = CartPoint{T}(0, 0, 0)

struct CylPoint{T<:Real}
    r::T
    phi::T
    z::T
end

CylPoint{T}() where {T} = CylPoint{T}(0, 0, 0)

mutable struct VelocityLookup
    e::Cfloat
    e100::Cfloat
    e110::Cfloat
    e111::Cfloat
    h100::Cfloat
    h110::Cfloat
    h111::Cfloat
    ea::Cfloat
    eb::Cfloat
    ec::Cfloat
    ebp::Cfloat
    ecp::Cfloat
    ha::Cfloat
    hb::Cfloat
    hc::Cfloat
    hbp::Cfloat
    hcp::Cfloat
    hcorr::Cfloat
    ecorr::Cfloat

    VelocityLookup() = new()
end

mutable struct SigGenSetup
    # general
    verbosity::Cint

    # geometry
    xtal_length::Cfloat
    xtal_radius::Cfloat
    pc_length::Cfloat
    pc_radius::Cfloat
    wrap_around_radius::Cfloat
    ditch_depth::Cfloat
    ditch_thickness::Cfloat
    bottom_taper_length::Cfloat
    hole_length::Cfloat
    hole_radius::Cfloat
    outer_taper_length::Cfloat
    taper_angle::Cfloat
    inner_taper_length::Cfloat
    outer_taper_width::Cfloat
    inner_taper_width::Cfloat
    top_bullet_radius::Cfloat
    bottom_bullet_radius::Cfloat
    Li_thickness::Cfloat

    # electric fields & weighing potentials
    xtal_grid::Cfloat
    impurity_z0::Cfloat
    impurity_gradient::Cfloat
    impurity_quadratic::Cfloat
    impurity_surface::Cfloat
    impurity_radial_add::Cfloat
    impurity_radial_mult::Cfloat
    impurity_rpower::Cfloat
    xtal_HV::Cfloat
    max_iterations::Cint
    write_field::Cint
    write_WP::Cint
    bulletize_PC::Cint

    # file names
    config_name::NTuple{256,Cchar}
    drift_name::NTuple{256,Cchar}
    field_name::NTuple{256,Cchar}
    wp_name::NTuple{256,Cchar}

    # signal calculation
    xtal_temp::Cfloat
    preamp_tau::Cfloat
    time_steps_calc::Cint
    step_time_calc::Cfloat
    step_time_out::Cfloat
    charge_cloud_size::Cfloat
    use_acceleration::Cint
    use_repulsion::Cint
    use_diffusion::Cint
    energy::Cfloat

    coord_type::Cint
    ntsteps_out::Cint

    # data for fields.c
    rmin::Cfloat
    rmax::Cfloat
    rstep::Cfloat
    zmin::Cfloat
    zmax::Cfloat
    zstep::Cfloat
    rlen::Cint
    zlen::Cint
    v_lookup_len::Cint
    v_lookup::Ptr{VelocityLookup}
    efld::Ptr{Ptr{CylPoint{Cfloat}}}
    wpot::Ptr{Ptr{Cfloat}}

    # data for calc_signal.c
    dpath_e::Ptr{CartPoint{Cfloat}}
    dpath_h::Ptr{CartPoint{Cfloat}}
    instant_vel_e::Ptr{CartPoint{Cfloat}}
    instant_vel_h::Ptr{CartPoint{Cfloat}}
    instant_charge_size_e::Ptr{Cfloat}
    instant_charge_size_h::Ptr{Cfloat}
    initial_vel::Cfloat
    final_vel::Cfloat
    dv_dE::Cfloat
    v_over_E::Cfloat
    final_charge_size::Cdouble

    function SigGenSetup()
        s = new()
        fill!(unsafe_wrap(Array, Ptr{UInt8}(pointer_from_objref(s)), sizeof(s)), 0)
        return s
    end
end

SigGenSetup(config_filename::AbstractString) = signal_calc_init(config_filename)

function tup2str(tup::NTuple{N,C}) where {N, C<:Union{AbstractChar, Cchar}}
    out = Vector{UInt8}(undef, N)

    for i in 1:N
        c = @inbounds tup[i]
        c == 0 && (resize!(out, i-1); break)
        @inbounds out[i] = tup[i]
    end

    return String(out)
end

function Base.getproperty(setup::SigGenSetup, sym::Symbol)
    if sym in (:config_name, :drift_name, :field_name, :wp_name)
        return tup2str(getfield(setup, sym))
    elseif sym in (:dpath_e, :dpath_h, :instant_vel_e, :instant_vel_h)
        ptr = getfield(setup, sym)
        N = setup.time_steps_calc
        v = unsafe_wrap(Vector{CartPoint{Cfloat}}, ptr, N)
        return transpose(reshape(reinterpret(Cfloat, v), (3, N)))
    elseif sym in (:instant_charge_size_e, :instant_charge_size_h)
        ptr = getfield(setup, sym)
        return unsafe_wrap(Vector{Cfloat}, ptr, setup.time_steps_calc)
    else
        return getfield(setup, sym)
    end
end


using Zlib

# 1-byte unsigned integer -> Uint64
function size1unpack(bytes::Array{Uint8,1})
    size1unpack(bytes[1])
end

function size1unpack(byte::Uint8)
    uint64(byte)
end

# 2-bytes unsigned integer in big endian format -> Uint64
function size2unpack(bytes::Array{Uint8,1})
    uint64(reinterpret(Uint16, reverse(bytes))[1])
end

# 4-bytes unsigned integer in big endian format -> Uint64
function size4unpack(bytes::Array{Uint8,1})
    uint64(reinterpret(Uint32, reverse(bytes))[1])
end

function int1unpack(bytes::Array{Uint8,1})
    int1unpack(bytes[1])
end

function int1unpack(byte::Uint8)
    int(byte)
end

function int4unpack(bytes)
    int(reinterpret(Int32, reverse(bytes))[1])
end

function floatunpack(bytes)
    reinterpret(Float64, reverse(bytes))[1]
end

function charintpack(value::Integer, size::Int)
    bytes = zeros(Uint8, size)
    for i=1:size
        bytes[i] = uint8(value)
        value = value >>> 8
        end
    reverse(bytes)
end

function charintpack(value::Integer)
    bytes = []
    while value != 0
        bytes = vcat(bytes, uint8(value & 0xff))
        value = value >>> 8
    end
    uint8(bytes)
end

function charint4pack(integer::Integer)
    charintpack(integer, 4)
end

function charint2pack(integer::Integer)
    charintpack(integer, 2)
end

function charsignedint4pack(integer::Integer)
    charintpack(integer, 4)
end

function lencheck(bytes::Array{Uint8,1}, limit::Int)
    len = uint64(length(bytes))
    lencheck(len, len < limit, bytes)
end

function lencheck(len::Uint64, limit::Uint64, bytes::Array{Uint8,1})
    lencheck(limit, len < limit, bytes)
end

function lencheck(len::Uint64, pred::Bool, bytes::Array{Uint8,1})
    if pred
        throw(IncompleteData(bytes))
    end
    len
end

function compressterm(encodedterm, compression::Bool)
    compressterm(encodedterm, 6)
end

function compressterm(encodedterm, compression::Int)
    if compression < 0 || compression > 9
        throw(InvalidCompressionLevel(compression))
    end
    comp = Zlib.compress(encodedterm, compression)
    len = length(encodedterm)
    # XXX add check here for too small of length
    if length(comp) + 5 <= len
        vcat(int4pack(len), comp)
    end
end

function decompressterm(bytes::Array{Uint8,1})
    if length(bytes) < 16
        throw(IncompleteData(bytes))
    end
    sentlen = size4unpack(bytes[3:6])
    term = Zlib.decompress(bytes[7:end])
    actuallen = length(term)
    if actuallen != sentlen
        msg = "Header declared $sentlen bytes but got $actuallen bytes."
        throw(InvalidCompressedTag(msg))
    end
    return term
end