getpvtestresult(learner, 300) |> svg
getpolicy(learner, getstate(Tac([2,8,6]))) |> display
getvalue(learner, getstate(Tac([2,8,6])), 1) |> display
getvalue(learner, getstate(Tac([2,8,6])), 3) |> display
getvalue(learner, getstate(Tac([2,8,6,9]))) |> display
getvalue(learner, getstate(Tac([2,8,6])), 9) |> display
getstate(Tac([2,8,6]))
mzg = play_game(learner, [2,8,6])
ts1 = TrainingSample(mzg, 1, 9, 9)


# STOPPED HERE
# The following two are not similar! (and 
# this is the main problem). UCB score is being badly
# effected by the bad recurrent_inference value results
# via the inferred value in backpropagate.
getvalue(learner, getstate(Tac([2,8,6,9,7,1,5, 4])))
getvalue(learner, getstate(Tac([2,8,6,9,7,1,5]), 4))




getstate(Tac([2,8,6])) |> display

value_fn = getvaluefn(Tac)
optimalp = getoptimalpolicyfn(Tac)

###########
s1 = getstate(Tac([2,8,6,1])) 
value_fn[s1]
p1 = getpolicy(learner, s1)
optimalp(s1)

s3 = getstate(Tac([2,8,6,3])) 
value_fn[s3]
p3 = getpolicy(learner, s3)
optimalp(s3)
###########

s4 = getstate(Tac([2,8,6,4]))
s4 |> display  
value_fn[s4]
p4 = getpolicy(learner, s4)
optimalp(s4)

s5 = getstate(Tac([2,8,6,5])) 
s5 |> display 
value_fn[s5]
p5 = getpolicy(learner, s5)
optimalp(s5)

s7 = getstate(Tac([2,8,6,7]))
s7 |> display 
value_fn[s7]
p7 = getpolicy(learner, s7)
optimalp(s7)

s9 = getstate(Tac([2,8,6,9])) 
s9 |> display 
value_fn[s9]
p9 = getpolicy(learner, s9)
optimalp(s9)













import Pkg
Pkg.rm(["Grids", "GridGames"])
Pkg.develop(["Grids", "GridGames"])

using MuZero
using GridGames
using Base.Threads: @spawn
cd(@__DIR__)
include("game.jl")
include("muzero_config.jl")
config = getconfig(Tac)
include("inference_tests.jl")
pv_tests = makepolicyvaluetests(Tac)
pv_tests.period = 100
learner = Learner(
    config;
    uuid="435c256d-1c60-4e07-ac03-a2430ae7b7bd",
    dir="~/.julia/dev/MuZeroModels/tictactoe/",
    tests=[pv_tests],
    schedule=tacschedule!
)
getpvtestresult(learner, 300)
getpvtestresult(learner, 1405) |> svg
length(learner.replay_buffers)
mzgame = learner.replay_buffers[1]
mzgame.game.states[end]
target = getstate(Tac([1,4,8,2])) #6,3,6
found = findmuzerogames(learner, target)

length(learner.replay_buffers)
learner.stats.policy_test_fails
#40 fails: 6,8,25,31,47,51,57,78,116,138,139,142,143,180,185,186,187,201,202,218,225,259,284,300,323,334,418,512,514,538,628,725,778,1098,1114,1188,1413,2120,2760,300
showresultsfull(learner; mean=100)
get_steps(learner)
acc, fails, value_mse = runtests(pv_tests, training_network(learner))

using Flux
test_net = new_test_network(learner.config)
testmode!.(get_nets(test_net), true)
updatetestparams!(learner.train, test_net)
testmode!.(get_nets(test_net), true)
acc2, fails2, value_mse2 = runtests(pv_tests, test_net)




#run all tests one by one! (instead of in batch mode)
fails = []
for pvtest in pv_tests
    out = initial_inference(test_net, pvtest.state) |> NetworkOutput
    append!(fails, getpolicyfails(pvtest, out))
end









fit!(learner; steps=500)
get_steps(learner)
learner.stats.policy_test_fails
# 45 fails: 6,25,31,47,51,57,78,116,121,128,131,138,139,142,172,180,185,187,201,218,225,259,284,323,334,418,512,514,538,628,725,761,778,839,1098,1114,1188,1318,1413,1606,2066,2120,2760,3254,300

getpvtestresult(learner, 3254) |> svg
target2 = getstate(Tac([1,5,4,6,2,9]))
found = findmuzerogames(
    learner, 
    getpolicyvaluetests(learner)[3254].state
)

fit!(learner; steps=90)
# 47 fails: 6,25,31,47,51,57,78,116,121,128,131,139,142,167,172,180,185,187,201,218,225,238,259,284,285,323,334,418,512,514,538,628,778,834,839,1098,1114,1188,1318,1413,1606,1861,2120,2760,3254,3286,300
found = findmuzerogames(
    learner, 
    getpolicyvaluetests(learner)[3286].state
)
getpolicy(
    learner,
    getpolicyvaluetests(learner)[1405].state
) |> display
getpvtestresult(learner, 3254) |> svg