TODOs:
navsim/agents/vadv2_map/vadv2_agent_pdm_progress_map.py
navsim/agents/vadv2_map/vadv2_pdm_model_progress_map.py
navsim/planning/script/config/common/agent/vadv2_map.yaml

# debug training in sleep ngc
agent=vadv2_map
python $NAVSIM_DEVKIT_ROOT/navsim/planning/script/run_training.py \
agent=$agent \
dataloader.params.batch_size=2 \
experiment_name=debug \
cache_path=null \
agent.config.ckpt_path=debug \
split=tiny \
scene_filter=navtiny

#========================== real training:==========================
single_node.sh / train_nodes.sh

# ==========================real inference:==========================
1. go to experiment dir, rename ckpts
for file in epoch=*-step=*.ckpt; do
  epoch=$(echo $file | sed -n 's/.*epoch=\([0-9][0-9]\).*/\1/p')
  new_filename="epoch${epoch}.ckpt"
  mv "$file" "$new_filename"
done

2. run evaluation
# multiple evaluation
agent="vadv2_8192_pdm_vit_mult0.1_progress_lw2"
dataset_cache="navtest_vadv2_4f_cache"
agent_ckpt=${agent}_ckpt
epochs=(0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29)
ckpts=(
    epoch00.ckpt epoch01.ckpt epoch02.ckpt epoch03.ckpt epoch04.ckpt epoch05.ckpt epoch06.ckpt epoch07.ckpt epoch08.ckpt epoch09.ckpt
    epoch10.ckpt epoch11.ckpt epoch12.ckpt epoch13.ckpt epoch14.ckpt epoch15.ckpt epoch16.ckpt epoch17.ckpt epoch18.ckpt epoch19.ckpt
    epoch20.ckpt epoch21.ckpt epoch22.ckpt epoch23.ckpt epoch24.ckpt epoch25.ckpt epoch26.ckpt epoch27.ckpt epoch28.ckpt epoch29.ckpt
)

inf_w=0.1;
inf_w_da=2.0
for i in {15..19}; do
    python ${NAVSIM_DEVKIT_ROOT}/navsim/planning/script/run_pdm_score_gpu.py \
        +use_pdm_closed=false \
        agent=$agent \
        +agent.config.inference_imi_weight=$inf_w \
        +agent.config.inference_da_weight=$inf_w_da \
        dataloader.params.batch_size=8 \
        worker.threads_per_node=64 \
        agent.checkpoint_path=${NAVSIM_EXP_ROOT}/${agent_ckpt}/${ckpts[$i]} \
        experiment_name=${agent_ckpt}/${epochs[$i]}_${inf_w}_da${inf_w_da}_xformers \
        +cache_path=${NAVSIM_EXP_ROOT}/${dataset_cache} \
        metric_cache_path=${NAVSIM_EXP_ROOT}/navtest_cache \
        split=test \
        scene_filter=navtest;
done

3. output results
for epoch in 10 11 12 13 14 15 16 17 18 19; do
imi_w=0.1
da_w=2.0
echo ===================${epoch}===================
cat $(find ./${epoch}_${imi_w}_da${da_w}_xformers/ -type f -name "*.csv") "end" | tail -n 1
# cat $(find ./${epoch}/ -type f -name "*.csv") "end" | tail -n 1
done