scripts/evaluation/eval_subscores.sh

# evaluate an epoch
# generate subscores for grid search: epochxx.pkl
# generate a temp PDM score for choosing the best epoch
agent=hydra_pe_temporal;
agent_ckpt=hydra_pe_temporal_vov_fixedpading_pe_temporal_modifyself_bs8x8_ckpt;
# 1. rename ckpts
cd ${NAVSIM_EXP_ROOT}/$agent_ckpt; 
for file in epoch=*-step=*.ckpt; do
  epoch=$(echo $file | sed -n 's/.*epoch=\([0-9][0-9]\).*/\1/p')
  new_filename="epoch${epoch}.ckpt"
  mv "$file" "$new_filename"
done
cd /navsim_ours;

# 2. eval
epochs=(0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24);
ckpts=(
    epoch00.ckpt epoch01.ckpt epoch02.ckpt epoch03.ckpt epoch04.ckpt epoch05.ckpt epoch06.ckpt epoch07.ckpt epoch08.ckpt epoch09.ckpt
    epoch10.ckpt epoch11.ckpt epoch12.ckpt epoch13.ckpt epoch14.ckpt epoch15.ckpt epoch16.ckpt epoch17.ckpt epoch18.ckpt epoch19.ckpt
    epoch20.ckpt epoch21.ckpt epoch22.ckpt epoch23.ckpt epoch24.ckpt
)


for i in {0..19}; do
    python ${NAVSIM_DEVKIT_ROOT}/navsim/planning/script/run_pdm_score_gpu.py \
        +use_pdm_closed=false \
        agent=$agent \
        dataloader.params.batch_size=8 \
        worker.threads_per_node=64 \
        agent.checkpoint_path=${NAVSIM_EXP_ROOT}/${agent_ckpt}/${ckpts[$i]} \
        experiment_name=${agent_ckpt}/${epochs[$i]}_xformers \
        +cache_path=null \
        metric_cache_path=${NAVSIM_EXP_ROOT}/navtest_cache \
        split=test \
        scene_filter=navtest;
done

# display scores
for epoch in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19; do

echo ===================${epoch}===================
cat $(find ./${epoch}_xformers/ -type f -name "*.csv") "end" | tail -n 1
done