Upload folder using huggingface_hub

.gitattributes

@@ -374,3 +374,148 @@ seq_10_150_3_3_encode_occ_separate_offsets_bs8_128_no_seqindex_long_lastvalid_ab
 seq_10_150_3_3_encode_occ_separate_offsets_bs8_128_no_seqindex_long_lastvalid_ablation_head/training_225391_e4a2ad39e6963273_insert_map.png filter=lfs diff=lfs merge=lfs -text
 seq_10_150_3_3_encode_occ_separate_offsets_bs8_128_no_seqindex_long_lastvalid_ablation_head/training_225391_f2e333d5ab085b71_occ_agent.png filter=lfs diff=lfs merge=lfs -text
 seq_10_150_3_3_encode_occ_separate_offsets_bs8_128_no_seqindex_long_lastvalid_ablation_head/training_225391_f2e333d5ab085b71_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_003352_12a725c99e2aaf56_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_003352_12a725c99e2aaf56_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_003353_11e7a18d6bb79688_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_003353_11f939421937f967_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_003353_14bdfe3b4ada19bf_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_003353_1561804123f0e337_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_003353_16d8494156a5b841_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_003353_174ad7295f45aa95_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_004263_115c6ba86bf683c5_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_004263_115c6ba86bf683c5_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_004263_13156353a84a2f2f_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_004263_13156353a84a2f2f_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_004263_164784242fd9b02f_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_004263_164784242fd9b02f_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_004263_169ca74c66bf9ed0_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_004263_169ca74c66bf9ed0_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_004263_17f8c4685566ccd9_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_004263_17f8c4685566ccd9_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_004263_1a33ad2fbb8602c1_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_004263_1a33ad2fbb8602c1_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_004263_1a6b7592d196519_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_022418_103c73c18d6a259c_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_022418_112386794f00764c_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_022418_112386794f00764c_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_022418_11c58fe26cfb6f48_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_022418_11c58fe26cfb6f48_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_022418_12dffd87c5ecb555_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_022418_12dffd87c5ecb555_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_022418_1706c66176c79f70_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_022418_1706c66176c79f70_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_022418_175bb0ed572c5d13_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_022418_175bb0ed572c5d13_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_022418_196c55cabdd20f0f_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_022418_196c55cabdd20f0f_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_024423_100bcd6931009e18_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_024423_100bcd6931009e18_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_024423_11505eaf54db8fc8_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_024423_11505eaf54db8fc8_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_024423_12c11ec584177c3c_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_024423_12c11ec584177c3c_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_024423_146810b3ef0dcdb5_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_024423_146810b3ef0dcdb5_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_024423_1697d9d2c3e3401_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_024423_1697d9d2c3e3401_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_024423_16dbbc77c7cab9cf_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_024423_16dbbc77c7cab9cf_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_024423_19406b81b5a3d8e3_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_028469_1151359d72de95c2_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_028469_1151359d72de95c2_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_028470_132b536b00ac8301_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_028470_1374468009268a65_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_028470_13c17a915498bbba_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_028470_14258fc85590d62e_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_028470_14f3bfcd16f87267_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_028470_16db1c89909ef67a_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_050998_14e86489c20dc4d1_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_050998_14e86489c20dc4d1_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_050998_167ea74ffca4c5b3_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_050998_167ea74ffca4c5b3_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_050998_17f377e79146934a_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_050998_17f377e79146934a_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_050998_18719f4bc78b1fb1_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_050998_18bea69fc559e74c_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_050998_18bea69fc559e74c_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_050998_18d5cc6231b661da_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_050998_18d5cc6231b661da_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_050998_191da1a2c274d4dd_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_050998_191da1a2c274d4dd_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_053003_109c6c00d4b7dce3_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_053003_10f41fa73efb08d6_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_053003_10f41fa73efb08d6_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_053003_131a3b84a3fdf224_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_053003_131a3b84a3fdf224_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_053003_14eea656f8bf4290_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_053003_14eea656f8bf4290_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_053003_15552a5885ba7411_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_053003_15552a5885ba7411_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_053003_15a85412f9461576_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_053003_15a85412f9461576_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_053003_19c4d85332f37fe0_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_053003_19c4d85332f37fe0_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_057049_1937fd973f91e037_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_057049_1937fd973f91e037_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_057050_1043f1243fe04991_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_057050_11cb5b2299c1ce61_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_057050_121bf46385c50fd9_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_057050_155afeea9a335564_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_057050_1645ceb5f33aae_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_057050_167ed97dc21f11e6_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_058906_1511234d1ff2dabe_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_058906_1511234d1ff2dabe_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_058907_10b1e58def1809c0_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_058907_1150482a7b59c61d_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_058907_1521384e8f6cfee4_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_058907_16264ec0155019b6_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_058907_1821beaf06a4d23b_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_058907_199c04b229ea182e_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_068353_10b09eacc06955d0_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_068353_1266bc553d3160ef_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_068353_13988e72e385dccf_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_068353_1451a415203c1913_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_068353_15c3e288411df31c_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_068353_1981d796620f25cc_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_068353_1a7c5d38a7dcefba_insert_map.png filter=lfs diff=lfs merge=lfs -text
+training_069263_117d7ae7722109b7_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_069263_117d7ae7722109b7_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_069263_141666bdcad3b052_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_069263_141666bdcad3b052_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_069263_14215cd0e72b589a_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_069263_14215cd0e72b589a_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_069263_1422363c5d2d7dca_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_069263_1422363c5d2d7dca_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_069263_1497a3857b6b26b3_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_069263_1497a3857b6b26b3_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_069263_14f5be1f643853ed_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_069263_14f5be1f643853ed_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_069263_1811d463770ffd2c_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_069263_1811d463770ffd2c_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_087418_1092a0e3ab0996d4_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_087418_1092a0e3ab0996d4_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_087418_1105a8fd1bca2238_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_087418_1105a8fd1bca2238_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_087418_11af6c3f74575313_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_087418_11af6c3f74575313_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_087418_12144349d2c57d3b_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_087418_132e9200a7993e14_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_087418_132e9200a7993e14_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_087418_16970bf716da04a3_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_087418_16970bf716da04a3_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_087418_183032bb514f0df5_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_087418_183032bb514f0df5_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_089423_11220a3845c87ed9_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_089423_11220a3845c87ed9_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_089423_1152dac4a37c9910_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_089423_1152dac4a37c9910_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_089423_1266bbf2f830888a_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_089423_1266bbf2f830888a_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_089423_149f682e19454efa_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_089423_149f682e19454efa_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_089423_187e3e8b968b540d_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_089423_187e3e8b968b540d_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_089423_19b3b2b054628318_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_089423_19b3b2b054628318_occ_pt.png filter=lfs diff=lfs merge=lfs -text
+training_089423_1a1988dca2bb96e2_occ_agent.png filter=lfs diff=lfs merge=lfs -text
+training_089423_1a1988dca2bb96e2_occ_pt.png filter=lfs diff=lfs merge=lfs -text

backups/configs/experiments/ablate_head_tokens.yaml

@@ -0,0 +1,107 @@
+# Config format schema number, the yaml support to valid case source from different dataset
+time_info: &time_info
+  num_historical_steps: 11
+  num_future_steps: 80
+  use_intention: True
+  token_size: 2048
+  predict_motion: True
+  predict_state: True
+  predict_map: True
+  predict_occ: True
+  state_token:
+    invalid: 0
+    valid: 1
+    enter: 2
+    exit: 3
+  pl2seed_radius: 75.
+  disable_head_token: True
+  grid_range: 150. # 2 times of pl2seed_radius
+  grid_interval: 3.
+  angle_interval: 3.
+  seed_size: 1
+  buffer_size: 128
+  max_num: 32
+
+Dataset:
+  root:
+  train_batch_size: 1
+  val_batch_size: 1
+  test_batch_size: 1
+  shuffle: True
+  num_workers: 1
+  pin_memory: True
+  persistent_workers: True
+  train_raw_dir: 'data/waymo_processed/training'
+  val_raw_dir: 'data/waymo_processed/validation'
+  test_raw_dir: 'data/waymo_processed/validation'
+  val_tfrecords_splitted: data/waymo_processed/training/validation_tfrecords_splitted
+  transform: WaymoTargetBuilder
+  train_processed_dir:
+  val_processed_dir:
+  test_processed_dir:
+  dataset: 'scalable'
+  <<: *time_info
+
+Trainer:
+  strategy: ddp_find_unused_parameters_false
+  accelerator: 'gpu'
+  devices: 1
+  max_epochs: 32
+  save_ckpt_path:
+  num_nodes: 1
+  mode:
+  ckpt_path:
+  precision: 32
+  accumulate_grad_batches: 1
+  overfit_epochs: 6000
+
+Model:
+  predictor: 'smart'
+  decoder_type: 'agent_decoder' # choose from ['agent_decoder', 'occ_decoder']
+  dataset: 'waymo'
+  input_dim: 2
+  hidden_dim: 128
+  output_dim: 2
+  output_head: False
+  num_heads: 8
+  <<: *time_info
+  head_dim: 16
+  dropout: 0.1
+  num_freq_bands: 64
+  lr: 0.0005
+  warmup_steps: 0
+  total_steps: 32
+  predict_map_token: False
+  num_recurrent_steps_val: 300
+  val_open_loop: False
+  val_close_loop: True
+  val_insert: False
+  n_rollout_close_val: 1
+  decoder:
+    <<: *time_info
+    num_map_layers: 3
+    num_agent_layers: 6
+    a2a_radius: 60
+    pl2pl_radius: 10
+    pl2a_radius: 30
+    a2sa_radius: 10
+    pl2sa_radius: 10
+    time_span: 60
+  loss_weight:
+    token_cls_loss: 1
+    map_token_loss: 1
+    state_cls_loss: 10
+    type_cls_loss: 5
+    pos_cls_loss: 1
+    head_cls_loss: 1
+    offset_reg_loss: 5
+    shape_reg_loss: .2
+    pos_reg_loss: 10
+    head_reg_loss: 10
+    state_weight: [0.1, 0.1, 0.8] # invalid, valid, exit
+    seed_state_weight: [0.1, 0.9] # invalid, enter
+    seed_type_weight: [0.8, 0.1, 0.1]
+    agent_occ_pos_weight: 100
+    pt_occ_pos_weight: 5
+    agent_occ_loss: 10
+    pt_occ_loss: 10

backups/configs/experiments/ablate_state_and_grid_tokens.yaml

@@ -0,0 +1,107 @@
+# Config format schema number, the yaml support to valid case source from different dataset
+time_info: &time_info
+  num_historical_steps: 11
+  num_future_steps: 80
+  use_intention: True
+  token_size: 2048
+  predict_motion: True
+  predict_state: True
+  predict_map: True
+  predict_occ: True
+  state_token:
+    invalid: 0
+    valid: 1
+    enter: 2
+    exit: 3
+  pl2seed_radius: 75.
+  disable_state_tokens: True
+  disable_grid_token: True
+  grid_range: 150. # 2 times of pl2seed_radius
+  grid_interval: 3.
+  angle_interval: 3.
+  seed_size: 1
+  buffer_size: 128
+  max_num: 32
+
+Dataset:
+  root:
+  train_batch_size: 1
+  val_batch_size: 1
+  test_batch_size: 1
+  shuffle: True
+  num_workers: 1
+  pin_memory: True
+  persistent_workers: True
+  train_raw_dir: 'data/waymo_processed/training'
+  val_raw_dir: 'data/waymo_processed/validation'
+  test_raw_dir: 'data/waymo_processed/validation'
+  val_tfrecords_splitted: data/waymo_processed/training/validation_tfrecords_splitted
+  transform: WaymoTargetBuilder
+  train_processed_dir:
+  val_processed_dir:
+  test_processed_dir:
+  dataset: 'scalable'
+  <<: *time_info
+
+Trainer:
+  strategy: ddp_find_unused_parameters_false
+  accelerator: 'gpu'
+  devices: 1
+  max_epochs: 32
+  save_ckpt_path:
+  num_nodes: 1
+  mode:
+  ckpt_path:
+  precision: 32
+  accumulate_grad_batches: 1
+  overfit_epochs: 6000
+
+Model:
+  predictor: 'smart'
+  decoder_type: 'agent_decoder' # choose from ['agent_decoder', 'occ_decoder']
+  dataset: 'waymo'
+  input_dim: 2
+  hidden_dim: 128
+  output_dim: 2
+  output_head: False
+  num_heads: 8
+  <<: *time_info
+  head_dim: 16
+  dropout: 0.1
+  num_freq_bands: 64
+  lr: 0.0005
+  warmup_steps: 0
+  total_steps: 32
+  predict_map_token: False
+  num_recurrent_steps_val: 300
+  val_open_loop: False
+  val_close_loop: True
+  val_insert: False
+  n_rollout_close_val: 1
+  decoder:
+    <<: *time_info
+    num_map_layers: 3
+    num_agent_layers: 6
+    a2a_radius: 60
+    pl2pl_radius: 10
+    pl2a_radius: 30
+    a2sa_radius: 10
+    pl2sa_radius: 10
+    time_span: 60
+  loss_weight:
+    token_cls_loss: 1
+    map_token_loss: 1
+    state_cls_loss: 10
+    type_cls_loss: 5
+    pos_cls_loss: 1
+    head_cls_loss: 1
+    offset_reg_loss: 5
+    shape_reg_loss: .2
+    pos_reg_loss: 10
+    state_weight: [0.1, 0.1, 0.8] # invalid, valid, exit
+    seed_state_weight: [0.1, 0.9] # invalid, enter
+    seed_type_weight: [0.8, 0.1, 0.1]
+    agent_occ_pos_weight: 100
+    pt_occ_pos_weight: 5
+    agent_occ_loss: 10
+    pt_occ_loss: 10

backups/configs/experiments/ablate_state_tokens.yaml

@@ -0,0 +1,106 @@
+# Config format schema number, the yaml support to valid case source from different dataset
+time_info: &time_info
+  num_historical_steps: 11
+  num_future_steps: 80
+  use_intention: True
+  token_size: 2048
+  predict_motion: True
+  predict_state: True
+  predict_map: True
+  predict_occ: True
+  state_token:
+    invalid: 0
+    valid: 1
+    enter: 2
+    exit: 3
+  pl2seed_radius: 75.
+  disable_state_tokens: True
+  grid_range: 150. # 2 times of pl2seed_radius
+  grid_interval: 3.
+  angle_interval: 3.
+  seed_size: 1
+  buffer_size: 128
+  max_num: 32
+
+Dataset:
+  root:
+  train_batch_size: 1
+  val_batch_size: 1
+  test_batch_size: 1
+  shuffle: True
+  num_workers: 1
+  pin_memory: True
+  persistent_workers: True
+  train_raw_dir: 'data/waymo_processed/training'
+  val_raw_dir: 'data/waymo_processed/validation'
+  test_raw_dir: 'data/waymo_processed/validation'
+  val_tfrecords_splitted: data/waymo_processed/training/validation_tfrecords_splitted
+  transform: WaymoTargetBuilder
+  train_processed_dir:
+  val_processed_dir:
+  test_processed_dir:
+  dataset: 'scalable'
+  <<: *time_info
+
+Trainer:
+  strategy: ddp_find_unused_parameters_false
+  accelerator: 'gpu'
+  devices: 1
+  max_epochs: 32
+  save_ckpt_path:
+  num_nodes: 1
+  mode:
+  ckpt_path:
+  precision: 32
+  accumulate_grad_batches: 1
+  overfit_epochs: 6000
+
+Model:
+  predictor: 'smart'
+  decoder_type: 'agent_decoder' # choose from ['agent_decoder', 'occ_decoder']
+  dataset: 'waymo'
+  input_dim: 2
+  hidden_dim: 128
+  output_dim: 2
+  output_head: False
+  num_heads: 8
+  <<: *time_info
+  head_dim: 16
+  dropout: 0.1
+  num_freq_bands: 64
+  lr: 0.0005
+  warmup_steps: 0
+  total_steps: 32
+  predict_map_token: False
+  num_recurrent_steps_val: 300
+  val_open_loop: False
+  val_close_loop: True
+  val_insert: False
+  n_rollout_close_val: 1
+  decoder:
+    <<: *time_info
+    num_map_layers: 3
+    num_agent_layers: 6
+    a2a_radius: 60
+    pl2pl_radius: 10
+    pl2a_radius: 30
+    a2sa_radius: 10
+    pl2sa_radius: 10
+    time_span: 60
+  loss_weight:
+    token_cls_loss: 1
+    map_token_loss: 1
+    state_cls_loss: 10
+    type_cls_loss: 5
+    pos_cls_loss: 1
+    head_cls_loss: 1
+    offset_reg_loss: 5
+    shape_reg_loss: .2
+    pos_reg_loss: 10
+    state_weight: [0.1, 0.1, 0.8] # invalid, valid, exit
+    seed_state_weight: [0.1, 0.9] # invalid, enter
+    seed_type_weight: [0.8, 0.1, 0.1]
+    agent_occ_pos_weight: 100
+    pt_occ_pos_weight: 5
+    agent_occ_loss: 10
+    pt_occ_loss: 10

backups/configs/ours_long_term_without_insertion.yaml

@@ -0,0 +1,106 @@
+# Config format schema number, the yaml support to valid case source from different dataset
+time_info: &time_info
+  num_historical_steps: 11
+  num_future_steps: 80
+  use_intention: True
+  token_size: 2048
+  predict_motion: True
+  predict_state: True
+  predict_map: True
+  predict_occ: True
+  state_token:
+    invalid: 0
+    valid: 1
+    enter: 2
+    exit: 3
+  pl2seed_radius: 75.
+  disable_insertion: True
+  grid_range: 150. # 2 times of pl2seed_radius
+  grid_interval: 3.
+  angle_interval: 3.
+  seed_size: 1
+  buffer_size: 128
+  max_num: 32
+
+Dataset:
+  root:
+  train_batch_size: 1
+  val_batch_size: 1
+  test_batch_size: 1
+  shuffle: True
+  num_workers: 1
+  pin_memory: True
+  persistent_workers: True
+  train_raw_dir: 'data/waymo_processed/training'
+  val_raw_dir: 'data/waymo_processed/validation'
+  test_raw_dir: 'data/waymo_processed/validation'
+  val_tfrecords_splitted: data/waymo_processed/training/validation_tfrecords_splitted
+  transform: WaymoTargetBuilder
+  train_processed_dir:
+  val_processed_dir:
+  test_processed_dir:
+  dataset: 'scalable'
+  <<: *time_info
+
+Trainer:
+  strategy: ddp_find_unused_parameters_false
+  accelerator: 'gpu'
+  devices: 1
+  max_epochs: 32
+  save_ckpt_path:
+  num_nodes: 1
+  mode:
+  ckpt_path:
+  precision: 32
+  accumulate_grad_batches: 1
+  overfit_epochs: 6000
+
+Model:
+  predictor: 'smart'
+  decoder_type: 'agent_decoder' # choose from ['agent_decoder', 'occ_decoder']
+  dataset: 'waymo'
+  input_dim: 2
+  hidden_dim: 128
+  output_dim: 2
+  output_head: False
+  num_heads: 8
+  <<: *time_info
+  head_dim: 16
+  dropout: 0.1
+  num_freq_bands: 64
+  lr: 0.0005
+  warmup_steps: 0
+  total_steps: 32
+  predict_map_token: False
+  num_recurrent_steps_val: 300
+  val_open_loop: False
+  val_close_loop: True
+  val_insert: False
+  n_rollout_close_val: 1
+  decoder:
+    <<: *time_info
+    num_map_layers: 3
+    num_agent_layers: 6
+    a2a_radius: 60
+    pl2pl_radius: 10
+    pl2a_radius: 30
+    a2sa_radius: 10
+    pl2sa_radius: 10
+    time_span: 60
+  loss_weight:
+    token_cls_loss: 1
+    map_token_loss: 1
+    state_cls_loss: 10
+    type_cls_loss: 5
+    pos_cls_loss: 1
+    head_cls_loss: 1
+    offset_reg_loss: 5
+    shape_reg_loss: .2
+    pos_reg_loss: 10
+    state_weight: [0.1, 0.1, 0.8] # invalid, valid, exit
+    seed_state_weight: [0.1, 0.9] # invalid, enter
+    seed_type_weight: [0.8, 0.1, 0.1]
+    agent_occ_pos_weight: 100
+    pt_occ_pos_weight: 5
+    agent_occ_loss: 10
+    pt_occ_loss: 10

backups/dev/datasets/scalable_dataset.py

@@ -70,7 +70,7 @@ class MultiDataset(Dataset):
         self._raw_files = list(tqdm(filter(lambda fn: fn.removesuffix('.pkl') in valid_scenarios, self._raw_files), leave=False))
         if len(self._raw_files) <= 0:
             raise RuntimeError(f'Invalid number of data {len(self._raw_files)}!')
-        self._raw_paths = list(map(lambda fn: os.path.join(raw_dir, fn), self._raw_files))
+        self._raw_paths = list(map(lambda fn: os.path.join(raw_dir, fn), self._raw_files))[:20000]
 
         self.logger.debug(f"The number of {split} dataset is {len(self._raw_paths)}")
         self.logger.debug(f"The buffer size is {self.buffer_size}")

backups/dev/metrics/compute_metrics.py

@@ -20,7 +20,7 @@ from argparse import ArgumentParser
 from torch import Tensor
 from google.protobuf import text_format
 from torchmetrics import Metric
-from typing import Optional, Sequence, List, Dict
+from typing import Optional, Sequence, List, Dict, Tuple
 
 from waymo_open_dataset.utils.sim_agents import submission_specs
 
@@ -480,6 +480,24 @@ def _compute_metametric(
     return metametric
 
 
+def _compute_metametric_long(
+    config: long_metrics_pb2.SimAgentMetricsConfig, # type: ignore
+    metrics: Dict[str, Tensor],
+):
+    """Computes the meta-metric aggregation."""
+    metametric = torch.zeros((metrics['linear_speed_likelihood'].shape[1]))
+    for field_name in _METRIC_FIELD_NAMES:
+        likelihood_field_name = field_name + '_likelihood'
+        weight = getattr(config, field_name).metametric_weight
+        metric_score = metrics[likelihood_field_name][0]
+        metametric += weight * metric_score
+    for field_name in _METRIC_FIELD_NAMES:
+        likelihood_field_name = field_name + '_likelihood'
+        metric_score = metrics[likelihood_field_name][0]
+        metametric[metric_score == 0] = 0.
+    return metametric
+
+
 @dataclasses.dataclass(frozen=True)
 class MetricFeatures:
 
@@ -709,11 +727,11 @@ class LogDistributions:
     distance_to_nearest_object: Tensor
     collision_indication: Tensor
     time_to_collision: Tensor
-    distance_to_road_edge: Tensor
     num_placement: Tensor
     num_removement: Tensor
     distance_placement: Tensor
     distance_removement: Tensor
+    distance_to_road_edge: Optional[Tensor] = None
     offroad_indication: Optional[Tensor] = None
 
 
@@ -751,13 +769,25 @@ def _reduce_average_with_validity(
     if tensor.shape != validity.shape:
         raise ValueError('Shapes of `tensor` and `validity` must be the same.'
                             f'(Actual: {tensor.shape}, {validity.shape}).')
-    cond_sum = torch.sum(torch.where(validity, tensor, torch.zeros_like(tensor)))
-    valid_sum = torch.sum(validity)
-    if valid_sum == 0:
-        return torch.tensor(0.)
+    cond_sum = torch.sum(torch.where(validity, tensor, torch.zeros_like(tensor)), dim=-1)
+    valid_sum = torch.sum(validity, dim=-1)
+    if valid_sum.sum() == 0:
+        return torch.full(valid_sum.shape[:2], -torch.inf)
     return cond_sum / valid_sum
 
 
+def _reduce_mean(tensor: Tensor, dim: Optional[int] = None) -> Tensor:
+    validity = (tensor > 0.) & (tensor <= 1.)
+    if dim is None:
+        sum = torch.sum(torch.where(validity, tensor, torch.zeros_like(tensor)))
+        count = validity.sum().clamp(min=1)
+        return sum / count
+    else:
+        sum = torch.sum(torch.where(validity, tensor, torch.zeros_like(tensor)), dim=0)
+        count = validity.sum(dim=0).clamp(min=1)
+        return sum / count
+
+
 def histogram_estimate(
     config: long_metrics_pb2.SimAgentMetricsConfig.HistogramEstimate, # type: ignore
     log_samples: Tensor,
@@ -889,7 +919,7 @@ def compute_scenario_metrics_for_bundle(
     log_distributions: LogDistributions,
     scenario_log: Optional[scenario_pb2.Scenario], # type: ignore
     scenario_rollouts: ScenarioRollouts,
-) -> long_metrics_pb2.SimAgentMetrics: # type: ignore
+) -> Tuple[long_metrics_pb2.SimAgentMetrics, dict]: # type: ignore
 
     features_fields = [field.name for field in dataclasses.fields(MetricFeatures)]
     features_fields.remove('object_id')
@@ -981,15 +1011,19 @@ def compute_scenario_metrics_for_bundle(
     )
     collision_likelihood = torch.exp(torch.mean(collision_score))
 
-    distance_to_objects_log_likelihodd = log_likelihood_estimate_timeseries(
+    distance_to_objects_log_likelihod = log_likelihood_estimate_timeseries(
         field='distance_to_nearest_object',
         feature_config=config.distance_to_nearest_object,
         sim_values=sim_features.distance_to_nearest_object,
         log_distributions=log_distributions.distance_to_nearest_object,
     )
-    distance_to_objects_likelihodd = torch.exp(_reduce_average_with_validity(
-        distance_to_objects_log_likelihodd, sim_features.valid))
-    # CONSOLE.log(f'distance_to_objects_likelihodd: {distance_to_objects_likelihodd}')
+    distance_to_objects_valid = sim_features.valid & (
+        (sim_features.distance_to_nearest_object >= config.distance_to_nearest_object.histogram.min_val) &
+        (sim_features.distance_to_nearest_object <= config.distance_to_nearest_object.histogram.max_val)
+    )
+    distance_to_objects_likelihod = torch.exp(_reduce_average_with_validity(
+        distance_to_objects_log_likelihod, distance_to_objects_valid))
+    # CONSOLE.log(f'distance_to_objects_likelihod: {distance_to_objects_likelihod}')
 
     ttc_log_likelihood = log_likelihood_estimate_timeseries(
         field='time_to_collision',
@@ -1041,7 +1075,7 @@ def compute_scenario_metrics_for_bundle(
         sim_values=sim_features.distance_placement,
         log_distributions=log_distributions.distance_placement,
     )
-    distance_placement_validity = (
+    distance_placement_validity = sim_features.valid.unfold(-1, SHIFT, SHIFT)[..., 0] & (
         (sim_features.distance_placement > config.distance_placement.histogram.min_val) &
         (sim_features.distance_placement < config.distance_placement.histogram.max_val)
     )
@@ -1053,7 +1087,7 @@ def compute_scenario_metrics_for_bundle(
         sim_values=sim_features.distance_removement,
         log_distributions=log_distributions.distance_removement,
     )
-    distance_removement_validity = (
+    distance_removement_validity = sim_features.valid.unfold(-1, SHIFT, SHIFT)[..., 0] & (
         (sim_features.distance_removement > config.distance_removement.histogram.min_val) &
         (sim_features.distance_removement < config.distance_removement.histogram.max_val)
     )
@@ -1071,24 +1105,43 @@ def compute_scenario_metrics_for_bundle(
 
     # ==== Meta metric ====
     likelihood_metrics = {
-        'linear_speed_likelihood': float(linear_speed_likelihood.numpy()),
-        'linear_acceleration_likelihood': float(linear_accel_likelihood.numpy()),
-        'angular_speed_likelihood': float(angular_speed_likelihood.numpy()),
-        'angular_acceleration_likelihood': float(angular_accel_likelihood.numpy()),
-        'distance_to_nearest_object_likelihood': float(distance_to_objects_likelihodd.numpy()),
-        'collision_indication_likelihood': float(collision_likelihood.numpy()),
-        'time_to_collision_likelihood': float(ttc_likelihood.numpy()),
-        # 'distance_to_road_edge_likelihoodfloat(': distance_road_edge_likelihood.nump)y(),
-        # 'offroad_indication_likelihoodfloat(': offroad_likelihood.nump)y(),
-        'num_placement_likelihood': float(num_placement_likelihood.numpy()),
-        'num_removement_likelihood': float(num_removement_likelihood.numpy()),
-        'distance_placement_likelihood': float(distance_placement_likelihood.numpy()),
-        'distance_removement_likelihood': float(distance_removement_likelihood.numpy()),
+        'linear_speed_likelihood': float(_reduce_mean(linear_speed_likelihood).numpy()),
+        'linear_acceleration_likelihood': float(_reduce_mean(linear_accel_likelihood).numpy()),
+        'angular_speed_likelihood': float(_reduce_mean(angular_speed_likelihood).numpy()),
+        'angular_acceleration_likelihood': float(_reduce_mean(angular_accel_likelihood).numpy()),
+        'distance_to_nearest_object_likelihood': float(_reduce_mean(distance_to_objects_likelihod).numpy()),
+        'collision_indication_likelihood': float(_reduce_mean(collision_likelihood).numpy()),
+        'time_to_collision_likelihood': float(_reduce_mean(ttc_likelihood).numpy()),
+        # 'distance_to_road_edge_likelihoodfloat(float(_reduce_mean(':).numpy()) distance_road_edge_likelihood.nump)y(),
+        # 'offroad_indication_likelihoodfloat(float(_reduce_mean(':).numpy()) offroad_likelihood.nump)y(),
+        'num_placement_likelihood': float(_reduce_mean(num_placement_likelihood).numpy()),
+        'num_removement_likelihood': float(_reduce_mean(num_removement_likelihood).numpy()),
+        'distance_placement_likelihood': float(_reduce_mean(distance_placement_likelihood).numpy()),
+        'distance_removement_likelihood': float(_reduce_mean(distance_removement_likelihood).numpy()),
+    }
+
+    likelihood_metrics_long = {
+        'linear_speed_likelihood': _reduce_mean(linear_speed_likelihood, dim=0).unsqueeze(dim=0),
+        'linear_acceleration_likelihood': _reduce_mean(linear_accel_likelihood, dim=0).unsqueeze(dim=0),
+        'angular_speed_likelihood': _reduce_mean(angular_speed_likelihood, dim=0).unsqueeze(dim=0),
+        'angular_acceleration_likelihood': _reduce_mean(angular_accel_likelihood, dim=0).unsqueeze(dim=0),
+        'distance_to_nearest_object_likelihood': _reduce_mean(distance_to_objects_likelihod, dim=0).unsqueeze(dim=0),
+        'collision_indication_likelihood': _reduce_mean(torch.exp(torch.mean(collision_score, dim=-1)), dim=0).unsqueeze(dim=0),
+        'time_to_collision_likelihood': _reduce_mean(ttc_likelihood, dim=0).unsqueeze(dim=0),
+        # 'distance_to_road_edge_likelihoodfloat(float(_reduce_mean(':).numpy()) distance_road_edge_likelihood.nump)y(),
+        # 'offroad_indication_likelihoodfloat(float(_reduce_mean(':).numpy()) offroad_likelihood.nump)y(),
+        'num_placement_likelihood': torch.exp(torch.mean(num_placement_log_likelihood, dim=-1)),
+        'num_removement_likelihood': torch.exp(torch.mean(num_removement_log_likelihood, dim=-1)),
+        'distance_placement_likelihood': _reduce_mean(distance_placement_likelihood, dim=0).unsqueeze(dim=0),
+        'distance_removement_likelihood': _reduce_mean(distance_removement_likelihood, dim=0).unsqueeze(dim=0),
     }
 
     metametric = _compute_metametric(
         config, long_metrics_pb2.SimAgentMetrics(**likelihood_metrics)
     )
+    metametric_long = _compute_metametric_long(
+        config, likelihood_metrics_long
+    )
     # CONSOLE.log(f'metametric: {metametric}')
 
     return long_metrics_pb2.SimAgentMetrics(
@@ -1097,6 +1150,10 @@ def compute_scenario_metrics_for_bundle(
         simulated_collision_rate=float(simulated_collision_rate.numpy()),
         # simulated_offroad_rate=simulated_offroad_rate.numpy(),
         **likelihood_metrics,
+    ), dict(
+        scenario_id=scenario_rollouts.scenario_id,
+        metametric=metametric_long.unsqueeze(dim=0),
+        **likelihood_metrics_long,
     )
 
 
@@ -1122,9 +1179,6 @@ def _get_log_distributions(
     """
     assert log_values.ndim == 2, f'Expect log_values.ndim==2, got {log_values.ndim}, shape {log_values.shape} for {field}' 
 
-    # [n_objects, n_steps] -> [n_objects * n_steps]
-    log_samples = log_values.reshape(-1)
-
     # ! estimate
     if estimate_method == 'histogram':
         config = feature_config.histogram
@@ -1135,16 +1189,22 @@ def _get_log_distributions(
                 additive_smoothing_pseudocount=feature_config.bernoulli.additive_smoothing_pseudocount
             )
         )
-        log_samples = log_samples.float()  # cast torch.bool to torch.float32
+        log_values = log_values.float()  # cast torch.bool to torch.float32
+
+    if 'distance_' in field:
+        log_values = log_values[(log_values > config.min_val) & (log_values < config.max_val)]
+
+    if field == 'num_placement':
+        log_values = log_values[:, :-2]  # ignore the last two steps
+
+    # [n_objects, n_steps] -> [n_objects * n_steps]
+    log_samples = log_values.reshape(-1)
 
     # We generate `num_bins`+1 edges for the histogram buckets.
     edges = torch.linspace(
         config.min_val, config.max_val, config.num_bins + 1
     ).float()
 
-    if field in ('distance_placement', 'distance_removement'):
-        log_samples = log_samples[(log_samples > config.min_val) & (log_samples < config.max_val)]
-
     # Clip the samples to avoid errors with histograms. Nonetheless, the min/max
     # values should be configured to never hit this condition in practice.
     log_samples = torch.clamp(log_samples, config.min_val, config.max_val)
@@ -1198,6 +1258,7 @@ class LongMetric(Metric):
         ]
         for k in self.field_names:
             self.add_state(k, default=torch.tensor(0.), dist_reduce_fx='sum')
+            self.add_state(f'{k}_long', default=[], dist_reduce_fx='cat')
         self.add_state('scenario_counter', default=torch.tensor(0.), dist_reduce_fx='sum')
         self.add_state('placement_valid_scenario_counter', default=torch.tensor(0.), dist_reduce_fx='sum')
         self.add_state('removement_valid_scenario_counter', default=torch.tensor(0.), dist_reduce_fx='sum')
@@ -1242,9 +1303,9 @@ class LongMetric(Metric):
             time_to_collision = _get_log_distributions('time_to_collision',
                 self.metrics_config.time_to_collision, self.log_features.time_to_collision,
             ),
-            distance_to_road_edge = _get_log_distributions('distance_to_road_edge',
-                self.metrics_config.distance_to_road_edge, self.log_features.distance_to_road_edge,
-            ),
+            # distance_to_road_edge = _get_log_distributions('distance_to_road_edge',
+            #     self.metrics_config.distance_to_road_edge, self.log_features.distance_to_road_edge,
+            # ),
             # dist_offroad_indication = _get_log_distributions(
             #     'offroad_indication',
             #     self.metrics_config.offroad_indication,
@@ -1259,12 +1320,10 @@ class LongMetric(Metric):
                 self.metrics_config.num_removement, self.log_features.num_removement.float(),
             ),
             distance_placement = _get_log_distributions('distance_placement',
-                self.metrics_config.distance_placement, (
-                    self.log_features.distance_placement[self.log_features.distance_placement > 0])[None, ...],
+                self.metrics_config.distance_placement, self.log_features.distance_placement,
             ),
             distance_removement = _get_log_distributions('distance_removement',
-                self.metrics_config.distance_removement, (
-                    self.log_features.distance_removement[self.log_features.distance_removement > 0])[None, ...],
+                self.metrics_config.distance_removement, self.log_features.distance_removement,
             ),
         )
 
@@ -1328,31 +1387,41 @@ class LongMetric(Metric):
             metrics = self.compute_metrics(outputs)
 
         for scenario_metrics in metrics:
+
+            _scenario_metrics, _scenario_metrics_long = scenario_metrics
+
             self.scenario_counter += 1
 
-            self.metametric += scenario_metrics.metametric
+            if _scenario_metrics.distance_placement_likelihood > 0:
+                self.placement_valid_scenario_counter += 1
+
+            if _scenario_metrics.distance_removement_likelihood > 0:
+                self.removement_valid_scenario_counter += 1
+
+            # float metrics
+            self.metametric += _scenario_metrics.metametric
             self.average_displacement_error += (
-                scenario_metrics.average_displacement_error
+                _scenario_metrics.average_displacement_error
             )
             self.min_average_displacement_error += (
-                scenario_metrics.min_average_displacement_error
+                _scenario_metrics.min_average_displacement_error
             )
-            self.linear_speed_likelihood += scenario_metrics.linear_speed_likelihood
+            self.linear_speed_likelihood += _scenario_metrics.linear_speed_likelihood
             self.linear_acceleration_likelihood += (
-                scenario_metrics.linear_acceleration_likelihood
+                _scenario_metrics.linear_acceleration_likelihood
             )
-            self.angular_speed_likelihood += scenario_metrics.angular_speed_likelihood
+            self.angular_speed_likelihood += _scenario_metrics.angular_speed_likelihood
             self.angular_acceleration_likelihood += (
-                scenario_metrics.angular_acceleration_likelihood
+                _scenario_metrics.angular_acceleration_likelihood
             )
             self.distance_to_nearest_object_likelihood += (
-                scenario_metrics.distance_to_nearest_object_likelihood
+                _scenario_metrics.distance_to_nearest_object_likelihood
             )
             self.collision_indication_likelihood += (
-                scenario_metrics.collision_indication_likelihood
+                _scenario_metrics.collision_indication_likelihood
             )
             self.time_to_collision_likelihood += (
-                scenario_metrics.time_to_collision_likelihood
+                _scenario_metrics.time_to_collision_likelihood
             )
             # self.distance_to_road_edge_likelihood += (
             #     scenario_metrics.distance_to_road_edge_likelihood
@@ -1360,37 +1429,74 @@ class LongMetric(Metric):
             # self.offroad_indication_likelihood += (
             #     scenario_metrics.offroad_indication_likelihood
             # )
-            self.simulated_collision_rate += scenario_metrics.simulated_collision_rate
+            self.simulated_collision_rate += _scenario_metrics.simulated_collision_rate
             # self.simulated_offroad_rate += scenario_metrics.simulated_offroad_rate
 
             self.num_placement_likelihood += (
-                scenario_metrics.num_placement_likelihood
+                _scenario_metrics.num_placement_likelihood
             )
             self.num_removement_likelihood += (
-                scenario_metrics.num_removement_likelihood
+                _scenario_metrics.num_removement_likelihood
             )
             self.distance_placement_likelihood += (
-                scenario_metrics.distance_placement_likelihood
+                _scenario_metrics.distance_placement_likelihood
             )
             self.distance_removement_likelihood += (
-                scenario_metrics.distance_removement_likelihood
+                _scenario_metrics.distance_removement_likelihood
             )
 
-            if scenario_metrics.distance_placement_likelihood > 0:
-                self.placement_valid_scenario_counter += 1
-
-            if scenario_metrics.distance_removement_likelihood > 0:
-                self.removement_valid_scenario_counter += 1
+            # long metrics
+            self.metametric_long.append(_scenario_metrics_long['metametric'])
+            self.linear_speed_likelihood_long.append(_scenario_metrics_long['linear_speed_likelihood'])
+            self.linear_acceleration_likelihood_long.append(
+                _scenario_metrics_long['linear_acceleration_likelihood']
+            )
+            self.angular_speed_likelihood_long.append(_scenario_metrics_long['angular_speed_likelihood'])
+            self.angular_acceleration_likelihood_long.append(
+                _scenario_metrics_long['angular_acceleration_likelihood']
+            )
+            self.distance_to_nearest_object_likelihood_long.append(
+                _scenario_metrics_long['distance_to_nearest_object_likelihood']
+            )
+            self.collision_indication_likelihood_long.append(
+                _scenario_metrics_long['collision_indication_likelihood']
+            )
+            self.time_to_collision_likelihood_long.append(
+                _scenario_metrics_long['time_to_collision_likelihood']
+            )
+            # self.distance_to_road_edge_likelihood += (
+            #     scenario_metrics.distance_to_road_edge_likelihood
+            # )
+            # self.offroad_indication_likelihood += (
+            #     scenario_metrics.offroad_indication_likelihood
+            # )
+            self.num_placement_likelihood_long.append(
+                _scenario_metrics_long['num_placement_likelihood']
+            )
+            self.num_removement_likelihood_long.append(
+                _scenario_metrics_long['num_removement_likelihood']
+            )
+            self.distance_placement_likelihood_long.append(
+                _scenario_metrics_long['distance_placement_likelihood']
+            )
+            self.distance_removement_likelihood_long.append(
+                _scenario_metrics_long['distance_removement_likelihood']
+            )
 
     def compute(self) -> Dict[str, Tensor]:
         metrics_dict = {}
+        metrics_long_dict = {}
         for k in self.field_names:
-            if k not in ('distance_placement', 'distance_removement'):
-                metrics_dict[k] = getattr(self, k) / self.scenario_counter
-            if k == 'distance_placement':
-                metrics_dict[k] = getattr(self, k) / self.placement_valid_scenario_counter
-            if k == 'distance_removement':
-                metrics_dict[k] = getattr(self, k) / self.removement_valid_scenario_counter
+            # float metrics
+            if k not in ('distance_placement_likelihood', 'distance_removement_likelihood'):
+                metrics_dict[k] = getattr(self, k) / max(self.scenario_counter, 1)
+            if k == 'distance_placement_likelihood':
+                metrics_dict[k] = getattr(self, k) / max(self.placement_valid_scenario_counter, 1)
+            if k == 'distance_removement_likelihood':
+                metrics_dict[k] = getattr(self, k) / max(self.removement_valid_scenario_counter, 1)
+            # long metrics
+            if len(getattr(self, f'{k}_long')) > 0:
+                metrics_long_dict[k] = _reduce_mean(torch.cat(getattr(self, f'{k}_long')), dim=0)
 
         mean_metrics = long_metrics_pb2.SimAgentMetrics(
             scenario_id='', **metrics_dict,
@@ -1398,7 +1504,10 @@ class LongMetric(Metric):
         final_metrics = self.aggregate_metrics_to_buckets(
             self.metrics_config, mean_metrics
         )
-        CONSOLE.log(f'final_metrics:\n{final_metrics}')
+        mean_long_metrics = metrics_long_dict
+        final_long_metrics = self.aggregate_metrics_long_to_buckets(
+            self.metrics_config, mean_long_metrics
+        )
 
         out_dict = {
             f"{self.prefix}/wosac/realism_meta_metric": final_metrics.realism_meta_metric,
@@ -1412,6 +1521,18 @@ class LongMetric(Metric):
         for k in self.field_names:
             out_dict[f"{self.prefix}/wosac_likelihood/{k}"] = float(metrics_dict[k])
 
+        out_dict.update({
+            f"{self.prefix}/wosac_long/realism_meta_metric": [round(x, 4) for x in final_long_metrics['realism_meta_metric'].tolist()],
+            f"{self.prefix}/wosac_long/kinematic_metrics": [round(x, 4) for x in final_long_metrics['kinematic_metrics'].tolist()],
+            f"{self.prefix}/wosac_long/interactive_metrics": [round(x, 4) for x in final_long_metrics['interactive_metrics'].tolist()],
+            f"{self.prefix}/wosac_long/map_based_metrics": [round(x, 4) for x in final_long_metrics['map_based_metrics'].tolist()],
+            f"{self.prefix}/wosac_long/placement_based_metrics": [round(x, 4) for x in final_long_metrics['placement_based_metrics'].tolist()],
+        })
+        for k in self.field_names:
+            if k not in metrics_long_dict:
+                continue
+            out_dict[f"{self.prefix}/wosac_long_likelihood/{k}"] = [round(x, 4) for x in metrics_long_dict[k].tolist()]
+
         return out_dict
 
     @staticmethod
@@ -1446,6 +1567,35 @@ class LongMetric(Metric):
                     simulated_offroad_rate=metrics.simulated_offroad_rate,
                 )
 
+    @staticmethod
+    def aggregate_metrics_long_to_buckets(
+        config: long_metrics_pb2.SimAgentMetricsConfig, # type: ignore
+        metrics: Dict[str, Tensor],
+    ) -> Dict[str, Tensor]:
+        """Aggregates metrics into buckets for better readability."""
+        bucketed_metrics = {}
+        for bucket_name, fields_in_bucket in _METRIC_FIELD_NAMES_BY_BUCKET.items():
+            weighted_metric, weights_sum = torch.zeros(metrics['linear_speed_likelihood'].shape[0]), 0.0
+            for field_name in fields_in_bucket:
+                likelihood_field_name = field_name + '_likelihood'
+                weight = getattr(config, field_name).metametric_weight
+                metric_score = metrics[likelihood_field_name]
+                weighted_metric += weight * metric_score
+                weights_sum += weight
+            if weights_sum == 0:
+                weights_sum = 1  # FIXME: hack!!!
+                # raise ValueError('The bucket\'s weight sum is zero. Check your metrics'
+                #                 ' config.')
+            bucketed_metrics[bucket_name] = weighted_metric / weights_sum
+
+        return dict(
+                    realism_meta_metric=metrics['metametric'],
+                    kinematic_metrics=bucketed_metrics['kinematic'],
+                    interactive_metrics=bucketed_metrics['interactive'],
+                    map_based_metrics=bucketed_metrics['map_based'],
+                    placement_based_metrics=bucketed_metrics['placement_based'],
+                )
+
     @staticmethod
     def load_metrics_config(config_path: str = 'dev/metrics/metric_config.textproto',
                             ) -> long_metrics_pb2.SimAgentMetricsConfig: # type: ignore
@@ -1788,7 +1938,8 @@ if __name__ == "__main__":
 
     elif args.debug:
 
-        debug_path = 'output/scalable_smart_long/validation_catk/idx_0_0_rollouts.pkl'
+        # debug_path = 'output/scalable_smart_long/validation_catk/idx_0_0_rollouts.pkl'
+        debug_path = '/u/xiuyu/work/dev4/dev/metrics/idx_0_0_rollouts.pkl'
 
         # ! for debugging
         with open(debug_path, 'rb') as f:

backups/dev/model/smart.py

@@ -21,6 +21,7 @@ from dev.datasets.preprocess import TokenProcessor
 from dev.metrics.compute_metrics import *
 from dev.utils.metrics import *
 from dev.utils.visualization import *
+from dev.utils.func import wrap_angle
 
 
 class SMART(pl.LightningModule):
@@ -66,6 +67,17 @@ class SMART(pl.LightningModule):
         if self.disable_grid_token:
             self.predict_occ = False
 
+        self.disable_head_token = getattr(model_config, 'disable_head_token') \
+            if hasattr(model_config, 'disable_head_token') else False
+        self.use_head_token = not self.disable_head_token
+
+        self.disable_state_token = getattr(model_config, 'disable_state_token') \
+            if hasattr(model_config, 'disable_state_token') else False
+        self.use_state_token = not self.disable_state_token
+
+        self.disable_insertion = getattr(model_config, 'disable_insertiion') \
+            if hasattr(model_config, 'disable_insertion') else False
+
         self.token_processer = TokenProcessor(self.token_size,
                                               training=self.training,
                                               predict_motion=self.predict_motion,
@@ -115,6 +127,9 @@ class SMART(pl.LightningModule):
             predict_occ=self.predict_occ,
             state_token=model_config.state_token,
             use_grid_token=self.use_grid_token,
+            use_head_token=self.use_head_token,
+            use_state_token=self.use_state_token,
+            disable_insertion=self.disable_insertion,
             seed_size=self.seed_size,
             buffer_size=model_config.decoder.buffer_size,
             num_recurrent_steps_val=model_config.num_recurrent_steps_val,
@@ -147,6 +162,7 @@ class SMART(pl.LightningModule):
             self.offset_reg_loss_seed = nn.MSELoss()
             self.shape_reg_loss_seed = nn.MSELoss()
             self.pos_reg_loss_seed = nn.MSELoss()
+            self.head_reg_loss_seed = nn.MSELoss()
         if self.predict_occ:
             self.occ_cls_loss = nn.CrossEntropyLoss()
             self.agent_occ_loss_seed = nn.BCEWithLogitsLoss(
@@ -370,6 +386,7 @@ class SMART(pl.LightningModule):
             next_shape_gt_seed      = pred['next_shape_gt_seed']
             next_state_eval_mask_seed = pred['next_state_eval_mask_seed']
             next_attr_eval_mask_seed = pred['next_attr_eval_mask_seed']
+            next_head_eval_mask_seed    = pred['next_head_eval_mask_seed']
 
             # when num_seed_gt=0 loss term will be NaN
             state_cls_loss_seed = self.state_cls_loss_seed(next_state_prob_seed[next_state_eval_mask_seed],
@@ -388,15 +405,12 @@ class SMART(pl.LightningModule):
 
             loss = loss + state_cls_loss_seed + type_cls_loss_seed + shape_reg_loss_seed
 
-            next_head_rel_prob_seed     = pred['next_head_rel_prob_seed']
-            next_head_rel_index_gt_seed = pred['next_head_rel_index_gt_seed']
-            next_offset_xy_seed         = pred['next_offset_xy_seed']
-            next_offset_xy_gt_seed      = pred['next_offset_xy_gt_seed']
-            next_head_eval_mask_seed    = pred['next_head_eval_mask_seed']
 
             if self.use_grid_token:
                 next_pos_rel_prob_seed      = pred['next_pos_rel_prob_seed']
                 next_pos_rel_index_gt_seed  = pred['next_pos_rel_index_gt_seed']
+                next_offset_xy_seed         = pred['next_offset_xy_seed']
+                next_offset_xy_gt_seed      = pred['next_offset_xy_gt_seed']
 
                 pos_cls_loss_seed = self.pos_cls_loss_seed(next_pos_rel_prob_seed[next_attr_eval_mask_seed],
                                                            next_pos_rel_index_gt_seed[next_attr_eval_mask_seed]) * self.loss_weight['pos_cls_loss']
@@ -415,13 +429,28 @@ class SMART(pl.LightningModule):
                                                            next_pos_rel_xy_gt_seed[next_attr_eval_mask_seed]) * self.loss_weight['pos_reg_loss']
                 pos_reg_loss_seed = torch.nan_to_num(pos_reg_loss_seed)
                 self.log('seed_pos_reg_loss', pos_reg_loss_seed, **log_params)
+
                 loss = loss + pos_reg_loss_seed
 
-            head_cls_loss_seed = self.head_cls_loss_seed(next_head_rel_prob_seed[next_head_eval_mask_seed],
-                                                         next_head_rel_index_gt_seed[next_head_eval_mask_seed]) * self.loss_weight['head_cls_loss']
-            self.log('seed_head_cls_loss', head_cls_loss_seed, **log_params)
+            if self.use_head_token:
+                next_head_rel_prob_seed     = pred['next_head_rel_prob_seed']
+                next_head_rel_index_gt_seed = pred['next_head_rel_index_gt_seed']
+
+                head_cls_loss_seed = self.head_cls_loss_seed(next_head_rel_prob_seed[next_head_eval_mask_seed],
+                                                             next_head_rel_index_gt_seed[next_head_eval_mask_seed]) * self.loss_weight['head_cls_loss']
+                self.log('seed_head_cls_loss', head_cls_loss_seed, **log_params)
 
-            loss = loss + head_cls_loss_seed
+                loss = loss + head_cls_loss_seed
+
+            else:
+                next_head_rel_theta_seed    = pred['next_head_rel_theta_seed']
+                next_head_rel_theta_gt_seed = pred['next_head_rel_theta_gt_seed']
+
+                head_reg_loss_seed = self.head_reg_loss_seed(next_head_rel_theta_seed[next_head_eval_mask_seed],
+                                                             next_head_rel_theta_gt_seed[next_head_eval_mask_seed]) * self.loss_weight['head_reg_loss']
+                self.log('seed_head_reg_loss', head_reg_loss_seed, **log_params)
+
+                loss = loss + head_reg_loss_seed
 
             # plot_scenario_ids = ['74ad7b76d5906d39', '1351ea8b8333ddcb', '1352066cc3c0508d', '135436833ce5b9e7', '13570a32432d449', '13577c32a81336fb']
             if random.random() < 4e-5 or int(os.getenv('DEBUG', 0)):
@@ -988,6 +1017,7 @@ class SMART(pl.LightningModule):
         data['agent']['bos_mask']           = torch.zeros_like(data['agent']['state_idx']).bool()
 
         data['agent']['pos_xy']     = torch.zeros_like(data['agent']['token_pos'])
+        data['agent']['heading_theta'] = torch.zeros_like(data['agent']['token_heading'])
         if self.predict_occ:
             num_step = data['agent']['state_idx'].shape[1]
             data['agent']['pt_grid_token_idx'] = torch.zeros_like(data['pt_token']['token_idx'])[None].repeat(num_step, 1).long()
@@ -1055,8 +1085,9 @@ class SMART(pl.LightningModule):
 
             data['agent']['grid_token_idx'][agent_batch_mask] = grid_token_idx
             data['agent']['grid_offset_xy'][agent_batch_mask] = offset_xy
-            data['agent']['pos_xy'][agent_batch_mask] = pos_xy
             data['agent']['heading_token_idx'][agent_batch_mask] = heading_token_idx
+            data['agent']['pos_xy'][agent_batch_mask] = pos_xy
+            data['agent']['heading_theta'][agent_batch_mask] = wrap_angle(rel_heading)
             data['agent']['sort_indices'][agent_batch_mask] = sort_indices
             data['agent']['inrange_mask'][agent_batch_mask] = torch.stack(is_inrange, dim=1)
             data['agent']['bos_mask'][agent_batch_mask] = torch.stack(is_bos, dim=1)

backups/dev/modules/agent_decoder.py

@@ -121,6 +121,9 @@ class SMARTAgentDecoder(nn.Module):
                  predict_occ: bool=False,
                  state_token: Dict[str, int]=None,
                  use_grid_token: bool=True,
+                 use_head_token: bool=True,
+                 use_state_token: bool=True,
+                 disable_insertion: bool=False,
                  seed_size: int=5,
                  buffer_size: int=32,
                  num_recurrent_steps_val: int=-1,
@@ -148,6 +151,9 @@ class SMARTAgentDecoder(nn.Module):
         self.predict_map = predict_map
         self.predict_occ = predict_occ
         self.use_grid_token = use_grid_token
+        self.use_head_token = use_head_token
+        self.use_state_token = use_state_token
+        self.disable_insertion = disable_insertion
         self.num_recurrent_steps_val = num_recurrent_steps_val
         self.loss_weight = loss_weight
         self.logger = logger
@@ -273,8 +279,12 @@ class SMARTAgentDecoder(nn.Module):
         else:
             self.seed_pos_rel_xy_predict_head = MLPLayer(input_dim=hidden_dim, hidden_dim=hidden_dim,
                                                          output_dim=2)
-        self.seed_heading_rel_token_predict_head = MLPLayer(input_dim=hidden_dim, hidden_dim=hidden_dim,
-                                                            output_dim=self.angle_size)
+        if self.use_head_token:
+            self.seed_heading_rel_token_predict_head = MLPLayer(input_dim=hidden_dim, hidden_dim=hidden_dim,
+                                                                output_dim=self.angle_size)
+        else:
+            self.seed_heading_rel_theta_predict_head = MLPLayer(input_dim=hidden_dim, hidden_dim=hidden_dim,
+                                                                output_dim=1)
         # self.seed_pt_occ_embed = MLPLayer(input_dim=self.grid_size, hidden_dim=hidden_dim,
         #                                   output_dim=hidden_dim)
 
@@ -1121,8 +1131,9 @@ class SMARTAgentDecoder(nn.Module):
 
         agent_grid_token_idx = data['agent']['grid_token_idx']
         agent_grid_offset_xy = data['agent']['grid_offset_xy']
-        agent_pos_xy = data['agent']['pos_xy']
         agent_head_token_idx = data['agent']['heading_token_idx']
+        agent_pos_xy = data['agent']['pos_xy']
+        agent_heading_theta = data['agent']['heading_theta']
         sort_indices = data['agent']['sort_indices']
 
         device = pos_a.device
@@ -1277,7 +1288,7 @@ class SMARTAgentDecoder(nn.Module):
             next_pos_rel_idx_seed = next_pos_rel_prob_seed.softmax(dim=-1).argmax(dim=-1, keepdim=True)
         else:
             next_pos_rel_prob_seed = self.seed_pos_rel_xy_predict_head(feat_seed)
-            next_pos_rel_xy_seed = F.tanh(next_pos_rel_prob_seed)
+            next_pos_rel_xy_seed = torch.tanh(next_pos_rel_prob_seed)
 
         next_pos_rel_index_gt = agent_grid_token_idx.long()
         next_pos_rel_xy_gt = agent_pos_xy.float() / self.pl2seed_radius
@@ -1364,17 +1375,24 @@ class SMARTAgentDecoder(nn.Module):
             feat_sa = self.a2a_attn_layers[i](feat_sa, r_a2sa, edge_index_a2sa)
             feat_sa = feat_sa.reshape(num_step, -1, self.hidden_dim).transpose(0, 1)
 
-        next_head_rel_prob_seed = self.seed_heading_rel_token_predict_head(feat_sa)
-        next_head_rel_idx_seed = next_head_rel_prob_seed.softmax(dim=-1).argmax(dim=-1, keepdim=True)
+        if self.use_head_token:
+            next_head_rel_theta_seed = None
+            next_head_rel_prob_seed = self.seed_heading_rel_token_predict_head(feat_sa)
+            next_head_rel_idx_seed = next_head_rel_prob_seed.softmax(dim=-1).argmax(dim=-1, keepdim=True)
+        else:
+            next_head_rel_prob_seed = None
+            next_head_rel_theta_seed = self.seed_heading_rel_theta_predict_head(feat_sa)
+            next_head_rel_theta_seed = torch.tanh(next_head_rel_theta_seed)[..., 0]
 
         next_head_rel_index_gt_seed = agent_head_token_idx.long()
+        next_head_rel_theta_gt_seed = agent_heading_theta.float() / torch.pi  # [-1, 1]
 
-        next_offset_xy_seed = next_offset_xy_gt_seed = None
+        next_offset_xy_seed = None
         if self.use_grid_token:
             next_offset_xy_seed = self.seed_offset_xy_predict_head(feat_sa)
             next_offset_xy_seed = torch.tanh(next_offset_xy_seed) * 2  # [-2, 2]
 
-            next_offset_xy_gt_seed = agent_grid_offset_xy.float()
+        next_offset_xy_gt_seed = agent_grid_offset_xy.float()
 
         # next token prediction mask
         bos_token_index = torch.nonzero(agent_state_index == self.enter_state)
@@ -1593,6 +1611,8 @@ class SMARTAgentDecoder(nn.Module):
                 'next_pos_rel_xy_gt_seed': next_pos_rel_xy_gt_seed,
                 'next_head_rel_prob_seed': next_head_rel_prob_seed,
                 'next_head_rel_index_gt_seed': next_head_rel_index_gt_seed,
+                'next_head_rel_theta_seed': next_head_rel_theta_seed,
+                'next_head_rel_theta_gt_seed': next_head_rel_theta_gt_seed,
                 'next_offset_xy_seed': next_offset_xy_seed,
                 'next_offset_xy_gt_seed': next_offset_xy_gt_seed,
                 'next_shape_seed': next_shape_seed,
@@ -1798,7 +1818,7 @@ class SMARTAgentDecoder(nn.Module):
             while True:
 
                 p += 1
-                if t == 0 or p - 1 >= insert_limit: break
+                if t == 0 or p - 1 >= insert_limit or self.disable_insertion: break
 
                 # rebuild inference mask since number of agents have changed
                 inference_mask = torch.zeros_like(temporal_mask)
@@ -1934,7 +1954,7 @@ class SMARTAgentDecoder(nn.Module):
                             continue
                 else:
                     next_pos_rel_xy_seed = self.seed_pos_rel_xy_predict_head(feat_seed[:, (self.num_historical_steps - 1) // self.shift - 1 + t])
-                    next_pos_seed = F.tanh(next_pos_rel_xy_seed) * self.pl2seed_radius + ego_pos_t_1
+                    next_pos_seed = torch.tanh(next_pos_rel_xy_seed) * self.pl2seed_radius + ego_pos_t_1
 
                 if torch.all(next_state_idx_seed == self.invalid_state) or num_new_agents + 1 > insert_limit:
                     break
@@ -2082,9 +2102,13 @@ class SMARTAgentDecoder(nn.Module):
                     feat_a = self.a2a_attn_layers[i](feat_a, r_a2sa, edge_index_a2sa)
                     feat_a = feat_a.reshape(num_infer_step, -1, self.hidden_dim).transpose(0, 1)
 
-                next_head_rel_prob_seed = self.seed_heading_rel_token_predict_head(feat_a[-num_new_agent:, (self.num_historical_steps - 1) // self.shift - 1 + t])
-                next_head_rel_idx_seed = next_head_rel_prob_seed.softmax(dim=-1).argmax(dim=-1, keepdim=True)
-                next_head_seed = wrap_angle(self.attr_tokenizer.decode_heading(next_head_rel_idx_seed) + ego_head_t_1)
+                if self.use_head_token:
+                    next_head_rel_prob_seed = self.seed_heading_rel_token_predict_head(feat_a[-num_new_agent:, (self.num_historical_steps - 1) // self.shift - 1 + t])
+                    next_head_rel_idx_seed = next_head_rel_prob_seed.softmax(dim=-1).argmax(dim=-1, keepdim=True)
+                    next_head_seed = wrap_angle(self.attr_tokenizer.decode_heading(next_head_rel_idx_seed) + ego_head_t_1)
+                else:
+                    next_head_rel_theta_seed = self.seed_heading_rel_theta_predict_head(feat_a[-num_new_agent:, (self.num_historical_steps - 1) // self.shift - 1 + t])
+                    next_head_seed = torch.tanh(next_head_rel_theta_seed) * torch.pi + ego_head_t_1
 
                 head_a[-num_new_agent:, (self.num_historical_steps - 1) // self.shift - 1 + t] = next_head_seed
 
@@ -2188,6 +2212,8 @@ class SMARTAgentDecoder(nn.Module):
             next_state_idx = next_state_prob.softmax(dim=-1).argmax(dim=-1)
             next_state_idx[next_state_idx == self.valid_state_type.index('exit')] = self.exit_state
             next_state_idx[av_index] = self.valid_state # force ego_agent to be valid
+            if not self.use_state_token:
+                next_state_idx[next_state_idx == self.exit_state] = self.valid_state
 
             # convert the predicted token to a 0.5s (6 timesteps) trajectory
             expanded_token_index = next_token_idx[..., None, None, None].expand(-1, -1, 6, 4, 2)
@@ -2302,6 +2328,7 @@ class SMARTAgentDecoder(nn.Module):
                 agent_grid_emb = self.grid_token_emb[grid_a]
                 feat_a = torch.cat([feat_a, agent_grid_emb], dim=-1)
             feat_a = self.fusion_emb(feat_a)
+            raw_feat_a = feat_a.clone()  # ! IMPORANT: need to update `raw_feat_a`
 
             next_token_idx_list.append(next_token_idx[:, None])
             next_state_idx_list.append(next_state_idx[:, None])

backups/dev/modules/smart_decoder.py

@@ -40,7 +40,10 @@ class SMARTDecoder(nn.Module):
                  predict_state: bool=False,
                  predict_map: bool=False,
                  predict_occ: bool=False,
-                 use_grid_token: bool=False,
+                 use_grid_token: bool=True,
+                 use_head_token: bool=True,
+                 use_state_token: bool=True,
+                 disable_insertion: bool=False,
                  state_token: Dict[str, int]=None,
                  seed_size: int=5,
                  buffer_size: int=32,
@@ -89,6 +92,9 @@ class SMARTDecoder(nn.Module):
             predict_occ=predict_occ,
             state_token=state_token,
             use_grid_token=use_grid_token,
+            use_head_token=use_head_token,
+            use_state_token=use_state_token,
+            disable_insertion=disable_insertion,
             seed_size=seed_size,
             buffer_size=buffer_size,
             num_recurrent_steps_val=num_recurrent_steps_val,

backups/dev/utils/visualization.py

@@ -7,6 +7,7 @@ import tensorflow as tf
 import numpy as np
 import numpy.typing as npt
 import fnmatch
+import io
 import seaborn as sns
 import matplotlib.axes as Axes
 import matplotlib.transforms as mtransforms
@@ -650,7 +651,7 @@ def plot_map_token(ax: Axes, map_points: npt.NDArray, token_pos: npt.NDArray, to
 
 @safe_run
 def plot_map(ax: Axes, map_points: npt.NDArray, color='black'):
-    ax.scatter(map_points[:, 0], map_points[:, 1], s=0.2, c=color, edgecolors='none')
+    ax.scatter(map_points[:, 0], map_points[:, 1], s=0.5, c=color, edgecolors='none')
 
     xmin = np.min(map_points[:, 0])
     xmax = np.max(map_points[:, 0])
@@ -744,36 +745,68 @@ def plot_all(map, xs, ys, angles, types, colors, is_avs, pl2seed_radius: float=2
     ax.grid(False)
     ax.set_aspect('equal', adjustable='box')
 
+    # ! set plot limit if need
+    if kwargs.get('limit_size', None):
+        cx = float(xs[is_avs])
+        cy = float(ys[is_avs])
+
+        lx, ly = kwargs['limit_size']
+        xmin, xmax = cx - lx, cx + lx
+        ymin, ymax = cy - ly, cy + ly
+
+        ax.set_xlim(xmin, xmax)
+        ax.set_ylim(ymin, ymax)
+
     # ax.legend(loc='best', frameon=True)
 
+    pil_image = None
     if kwargs.get('save_path', None):
         plt.savefig(kwargs['save_path'], dpi=600, bbox_inches="tight")
 
+    else:
+        # ！convert to PIL image
+        buf = io.BytesIO()
+        plt.savefig(buf, format='png', dpi=600, bbox_inches='tight')
+        buf.seek(0)
+        pil_image = Image.open(buf).convert('RGB')
+
     plt.close()
 
-    return ax
+    return pil_image
 
 
 @safe_run
 def plot_file(gt_folder: str,
               folder: Optional[str] = None,
-              files: Optional[str] = None):
+              files: Optional[str] = None,
+              save_gif: bool = True,
+              batch_idx: Optional[int] = None,
+              time_idx: Optional[List[int]] = None,
+              limit_size: Optional[List[int]] = None,
+              **kwargs,
+    ) -> List[Image.Image]:
+
     from dev.metrics.compute_metrics import _unbatch
 
+    shift = 5
+
     if files is None:
         assert os.path.exists(folder), f'Path {folder} does not exist.'
         files = list(fnmatch.filter(os.listdir(folder), 'idx_*_rollouts.pkl'))
         CONSOLE.log(f'Found {len(files)} rollouts files from {folder}.')
 
-
     if folder is None:
         assert os.path.exists(files), f'Path {files} does not exist.'
         folder = os.path.dirname(files)
         files = [files]
 
     parent, folder_name = os.path.split(folder.rstrip(os.sep))
-    save_path = os.path.join(parent, f'{folder_name}_plots')
+    if save_gif:
+        save_path = os.path.join(parent, f'{folder_name}_plots')
+    else:
+        save_path = None
 
+    file_outs = []
     for file in (pbar := tqdm(files, leave=False, desc='Plotting files ...')):
         pbar.set_postfix(file=file)
 
@@ -786,25 +819,64 @@ def plot_file(gt_folder: str,
         preds_traj = _unbatch(preds['pred_traj'], agent_batch)
         preds_head = _unbatch(preds['pred_head'], agent_batch)
         preds_type = _unbatch(preds['pred_type'], agent_batch)
-        preds_state = _unbatch(preds['pred_state'], agent_batch)
+        if 'pred_state' in preds:
+            preds_state = _unbatch(preds['pred_state'], agent_batch)
+        else:
+            preds_state = tuple([torch.ones((*traj.shape[:2], traj.shape[2] // shift)) for traj in preds_traj])  # [n_agent, n_rollout, n_step2Hz]
         preds_valid = _unbatch(preds['pred_valid'], agent_batch)
 
+        # ! fetch certain scenario
+        if batch_idx is not None:
+            scenario_ids = scenario_ids[batch_idx : batch_idx + 1]
+            agent_id = (agent_id[batch_idx],)
+            preds_traj = (preds_traj[batch_idx],)
+            preds_head = (preds_head[batch_idx],)
+            preds_type = (preds_type[batch_idx],)
+            preds_state = (preds_state[batch_idx],)
+            preds_valid = (preds_valid[batch_idx],)
+
+        scenario_outs = []
         for i, scenario_id in enumerate(scenario_ids):
-            n_rollouts = preds_traj[0].shape[1]
+            n_agent, n_rollouts = preds_traj[0].shape[:2]
 
+            rollout_outs = []
             for j in range(n_rollouts):  # 1
                 pred = dict(scenario_id=[scenario_id],
                             pred_traj=preds_traj[i][:, j],
                             pred_head=preds_head[i][:, j],
-                            pred_state=preds_state[i][:, j],
+                            pred_state=(
+                                torch.cat([torch.zeros(n_agent, 1), preds_state[i][:, j].repeat_interleave(repeats=shift, dim=-1)],
+                                          dim=1)
+                            ),
                             pred_type=preds_type[i][:, j],
                     )
-                av_index = agent_id[i][:, 0].tolist().index(preds['av_id'])  # NOTE: hard code!!!
 
+                # NOTE: hard code!!!
+                if 'av_id' in preds:
+                    av_index = agent_id[i][:, 0].tolist().index(preds['av_id'])
+                else:
+                    av_index = n_agent - 1
+
+                # ! load logged data
                 data_path = os.path.join(gt_folder, 'validation', f'{scenario_id}.pkl')
                 with open(data_path, 'rb') as f:
                     data = pickle.load(f)
-                plot_val(data, pred, av_index=av_index, save_path=save_path)
+
+                rollout_outs.append(
+                    plot_val(data, pred,
+                             av_index=av_index,
+                             save_path=save_path,
+                             save_gif=save_gif,
+                             time_idx=time_idx,
+                             limit_size=limit_size,
+                             **kwargs
+                    )
+                )
+
+            scenario_outs.append(rollout_outs)
+        file_outs.append(scenario_outs)
+
+    return file_outs
 
 
 @safe_run
@@ -829,9 +901,9 @@ def plot_val(data: Union[dict, str], pred: dict, av_index: int, save_path: str,
     if 'agent_labels' in pred:
         kwargs.update(agent_labels=pred['agent_labels'])
 
-    plot_scenario(scenario_id, map_point, pred_traj, pred_head, pred_state, pred_type,
-                  av_index=av_index, ids=ids, save_path=save_path, suffix=suffix,
-                  pl2seed_radius=pl2seed_radius, attr_tokenizer=attr_tokenizer, **kwargs)
+    return plot_scenario(scenario_id, map_point, pred_traj, pred_head, pred_state, pred_type,
+                         av_index=av_index, ids=ids, save_path=save_path, suffix=suffix,
+                         pl2seed_radius=pl2seed_radius, attr_tokenizer=attr_tokenizer, **kwargs)
 
 
 @safe_run
@@ -842,7 +914,7 @@ def plot_scenario(scenario_id: str,
                   state: npt.NDArray,
                   types: List[str],
                   av_index: int,
-                  color_type: Literal['state', 'type', 'seed']='seed',
+                  color_type: Literal['state', 'type', 'seed', 'insert']='seed',
                   state_type: List[str]=['invalid', 'valid', 'enter', 'exit'],
                   plot_enter: bool=False,
                   suffix: str='',
@@ -862,7 +934,11 @@ def plot_scenario(scenario_id: str,
         num_historical_steps = 2
         shift = 1
 
-    if 'save_path' in kwargs and kwargs['save_path'] != '':
+    if (
+        'save_path' in kwargs
+        and kwargs['save_path'] != ''
+        and kwargs['save_path'] != None
+    ):
         os.makedirs(kwargs['save_path'], exist_ok=True)
         save_id = int(max([0] + list(map(lambda fname: int(fname.split("_")[-1]),
                                         filter(lambda fname: fname.startswith(scenario_id)
@@ -909,13 +985,30 @@ def plot_scenario(scenario_id: str,
         agent_colors[is_exited, :] = seed_colors['exited']
         agent_colors[is_entered, :] = seed_colors['entered']
 
+    if color_type == 'insert':
+        agent_colors[:, :] = seed_colors['exited']
+        agent_colors[av_index + 1:] = seed_colors['existing']
+
     agent_colors[av_index, :] = np.array(agent_palette[-1])
     is_av = np.zeros_like(state[:, 0]).astype(np.bool_)
     is_av[av_index] = True
 
-    # draw agents
+    # ! get timesteps to plot
+    timesteps = list(range(num_timestep))
+    if kwargs.get('time_idx', None) is not None:
+        time_idx = kwargs['time_idx']
+        assert set(time_idx).issubset(set(timesteps)), f'Got invalid time_idx: {time_idx=} v.s. {timesteps=}'
+        timesteps = sorted(time_idx)
+
+    # ! get plot limits
+    limit_size = kwargs.get('limit_size', None)
+    if limit_size is not None:
+        assert len(limit_size) == 2, f'Got invalid `limit_size`: {limit_size=}'
+
+    # ! plot all
+    pil_images = []
     fig_paths = []
-    for tid in tqdm(range(num_timestep), leave=False, desc="Plot ..."):
+    for tid in tqdm(timesteps, leave=False, desc="Plot ..."):
         mask_t = visible_mask[:, tid]
         xs = traj[mask_t, tid, 0]
         ys = traj[mask_t, tid, 1]
@@ -931,13 +1024,23 @@ def plot_scenario(scenario_id: str,
             labels = [agent_labels[i][tid // shift] for i in range(len(agent_labels)) if mask_t[i]]
 
         fig_path = None
-        if 'save_path' in kwargs:
+        if kwargs.get('save_path', None) is not None:
             save_path = kwargs['save_path']
             fig_path = os.path.join(f"{save_path}/{scenario_id}_{str(save_id).zfill(3)}", f"{tid}.png")
             fig_paths.append(fig_path)
 
-        plot_all(map_data, xs, ys, angles, types_t, colors=colors, save_path=fig_path, is_avs=is_av_t,
-                 pl2seed_radius=pl2seed_radius, attr_tokenizer=attr_tokenizer, enter_index=enter_index_t, labels=labels)
+        pil_images.append(
+            plot_all(map_data, xs, ys, angles, types_t,
+                     colors=colors,
+                     save_path=fig_path,
+                     is_avs=is_av_t,
+                     pl2seed_radius=pl2seed_radius,
+                     attr_tokenizer=attr_tokenizer,
+                     enter_index=enter_index_t,
+                     labels=labels,
+                     limit_size=limit_size,
+            )
+        )
 
     # generate gif
     if fig_paths and save_gif:
@@ -958,6 +1061,8 @@ def plot_scenario(scenario_id: str,
         except Exception as e:
             tqdm.write(f"{e}! Failed to save gif at {gif_path}")
 
+    return pil_images
+
 
 def match_token_map(data):
 
@@ -1096,19 +1201,132 @@ def plot_tokenize(data, save_path: str):
     agent_type = agent_data['type']
     ids = np.arange(raw_traj.shape[0])
 
-    plot_scenario(scenario_id=tokenized_data['scenario_id'],
-                  map_data=tokenized_data['map_point']['position'].numpy(),
-                  traj=raw_traj.numpy(),
-                  heading=raw_heading.numpy(),
-                  state=agent_state.numpy(),
-                  types=list(map(lambda i: AGENT_TYPE[i], agent_type.tolist())),
-                  av_index=av_index,
-                  ids=ids,
-                  save_path=save_path,
-                  pl2seed_radius=pl2seed_radius,
-                  attr_tokenizer=attr_tokenizer,
-                  color_type='state',
-                  )
+    return plot_scenario(
+                scenario_id=tokenized_data['scenario_id'],
+                map_data=tokenized_data['map_point']['position'].numpy(),
+                traj=raw_traj.numpy(),
+                heading=raw_heading.numpy(),
+                state=agent_state.numpy(),
+                types=list(map(lambda i: AGENT_TYPE[i], agent_type.tolist())),
+                av_index=av_index,
+                ids=ids,
+                save_path=save_path,
+                pl2seed_radius=pl2seed_radius,
+                attr_tokenizer=attr_tokenizer,
+                color_type='state',
+        )
+
+
+def get_metainfos(folder: str):
+
+    import pandas as pd
+
+    assert os.path.exists(folder), f'Path {folder} does not exist.'
+    files = list(fnmatch.filter(os.listdir(folder), 'idx_*_rollouts.pkl'))
+    CONSOLE.log(f'Found {len(files)} rollouts files from {folder}.')
+
+    metainfos_path = f'{os.path.normpath(folder)}_metainfos.parquet'
+    csv_path = f'{os.path.normpath(folder)}_metainfos.csv'
+
+    if not os.path.exists(metainfos_path):
+
+        data = []
+        for file in tqdm(files):
+            pkl_data = pickle.load(open(os.path.join(folder, file), 'rb'))
+            data.extend((file, scenario_id, index) for index, scenario_id in enumerate(pkl_data['_scenario_id']))
+
+        df = pd.DataFrame(data, columns=('rollout_file', 'scenario_id', 'index'))
+        df.to_parquet(metainfos_path)
+        df.to_csv(csv_path)
+        CONSOLE.log(f'Successfully saved to {metainfos_path}.')
+
+    else:
+        CONSOLE.log(f'File {metainfos_path} already exists!')
+        return
+
+
+def plot_comparison(methods: List[str], rollouts_paths: List[str], gt_folders: List[str],
+                    save_path: str, scenario_ids: Optional[List[str]] = None):
+    import pandas as pd
+    from collections import defaultdict
+
+    # ! hyperparameter
+    fps = 10
+
+    plot_time = [1, 6, 12, 18, 24, 30]
+    # plot_time = [1, 5, 10, 15, 20, 25]
+    time_idx = [int(time * fps) for time in plot_time]
+
+    limit_size = [75, 60]  # [width, height]
+
+    # ! load metainfos
+    metainfos = defaultdict(dict)
+    for method, rollout_path in zip(methods, rollouts_paths):
+        meta_info_path = f'{os.path.normpath(rollout_path)}_metainfos.parquet'
+        metainfos[method]['df'] = pd.read_parquet(meta_info_path)
+        CONSOLE.log(f'Loaded {method=} with {len(metainfos[method]["df"]["scenario_id"])=}.')
+    common_scenarios = set(metainfos['ours']['df']['scenario_id'])
+    for method, meta_info in metainfos.items():
+        if method == 'ours':
+            continue
+        common_scenarios &= set(meta_info['df']['scenario_id'])
+    for method, meta_info in metainfos.items():
+        df = metainfos[method]['df']
+        metainfos[method]['df'] = df[df['scenario_id'].isin(common_scenarios)]
+    CONSOLE.log(f'Filter and get {len(common_scenarios)=}.')
+
+    # ! load data and plot
+    if scenario_ids is None:
+        scenario_ids = metainfos['ours']['df']['scenario_id'].tolist()
+    CONSOLE.log(f'Plotting {len(scenario_ids)=} ...')
+
+    for scenario_id in (pbar := tqdm(scenario_ids)):
+        pbar.set_postfix(scenario_id=scenario_id)
+
+        figures = dict()
+        for method, rollout_path, gt_folder in zip(methods, rollouts_paths, gt_folders):
+            df = metainfos[method]['df']
+            _df = df.loc[df['scenario_id'] == scenario_id]
+            batch_idx = int(_df['index'].tolist()[0])
+            rollout_file = _df['rollout_file'].tolist()[0]
+            figures[method] = plot_file(
+                                gt_folder=gt_folder,
+                                files=os.path.join(rollout_path, rollout_file),
+                                save_gif=False,
+                                batch_idx=batch_idx,
+                                time_idx=time_idx,
+                                limit_size=limit_size,
+                                color_type='insert',
+                            )[0][0][0]
+
+        # ! plot figures
+        border = 5
+        padding_x = 20
+        padding_y = 50
+
+        img_width, img_height = figures['ours'][0].size
+        img_width = img_width + 2 * border
+        img_height = img_height + 2 * border
+        n_col = len(time_idx)
+        n_row = len(methods)
+
+        W = n_col * img_width + (n_col - 1) * padding_x
+        H = n_row * img_height + (n_row - 1) * padding_y
+
+        canvas = Image.new('RGB', (W, H), 'white')
+        for i_row, (method, method_figures) in enumerate(figures.items()):
+            for i_col, method_figure in enumerate(method_figures):
+                x = i_col * (img_width + padding_x)
+                y = i_row * (img_height + padding_y)
+
+                padded_figure = Image.new('RGB', (img_width, img_height), 'black')
+                padded_figure.paste(method_figure, (border, border))
+
+                canvas.paste(padded_figure, (x, y))
+
+        canvas.save(
+            os.path.join(save_path, f'{scenario_id}.png')
+        )
 
 
 if __name__ == "__main__":
@@ -1124,6 +1342,11 @@ if __name__ == "__main__":
     parser.add_argument('--plot_file', action='store_true')
     parser.add_argument('--folder_path', type=str, default=None)
     parser.add_argument('--file_path', type=str, default=None)
+    # metainfos
+    parser.add_argument('--get_metainfos', action='store_true')
+    # plot comparison
+    parser.add_argument('--plot_comparison', action='store_true')
+    parser.add_argument('--comparison_folder', type=str, default='comparisons')
     args = parser.parse_args()
 
     if args.plot_tokenize:
@@ -1143,3 +1366,29 @@ if __name__ == "__main__":
     if args.plot_file:
 
         plot_file(args.data_path, folder=args.folder_path, files=args.file_path)
+
+    if args.get_metainfos:
+
+        assert args.folder_path is not None, f'`folder_path` should not be None!'
+        get_metainfos(args.folder_path)
+
+    if args.plot_comparison:
+
+        methods = ['ours', 'smart']
+        gt_folders = [
+            '/baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/work/dev4/data/processed',
+            '/baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/work/dev4/data/processed',
+            '/baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/work/dev4/data/processed',
+        ]
+        rollouts_paths = [
+            '/baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/work/dev4/output/scalable_smart_long/validation_ours0',
+            '/baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/work/dev4/output/scalable_smart_long/validation_smart',
+            '/baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/work/dev4/output/scalable_smart_long/validation_cslft',
+        ]
+        save_path = f'/baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/work/dev4/output/scalable_smart_long/{args.comparison_folder}/'
+        os.makedirs(save_path, exist_ok=True)
+
+        scenario_ids = ['72ff3e1540b28431','a16c927b1a1cca74','a504d55ea6658de7','639949ea1d16125b']
+        plot_comparison(methods, rollouts_paths, gt_folders,
+                        save_path=save_path,
+                        scenario_ids=scenario_ids)

backups/scripts/compute_metrics.sh

@@ -1,7 +1,20 @@
 #!/bin/bash
 
+# network
+source /baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/clash.sh
+bash /baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/clash-for-linux-backup/start.sh
+proxy_on
+
+# env
+source /baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/anaconda3/etc/profile.d/conda.sh
+conda config --append envs_dirs /baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/.conda/envs
+conda activate traj
+
+cd /baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/work/dev4/
+
 export TORCH_LOGS='0'
 export TF_CPP_MIN_LOG_LEVEL='2'
+export TF_ENABLE_ONEDNN_OPTS='0'
 export PYTHONPATH='.'
 
 NUM_WORKERS=$1
@@ -10,4 +23,4 @@ SIM_DIR=$2
 echo 'Start running ...'
 python dev/metrics/compute_metrics.py --compute_metric --num_workers "$NUM_WORKERS" --sim_dir "$SIM_DIR" ${@:3}
 
-echo 'Done!
+echo 'Done!

backups/scripts/g8.sh

@@ -8,7 +8,7 @@
 #SBATCH --nodes=1                       # Single node or multi node
 #SBATCH --nodelist=sota-6
 #SBATCH --time 120:00:00                 # Max time (hh:mm:ss)
-#SBATCH --gres=gpu:8                    # GPUs per node
+#SBATCH --gres=gpu:7                    # GPUs per node
 #SBATCH --mem=256G                      # Recommend 32G per GPU
 #SBATCH --ntasks-per-node=1             # Tasks per node
 #SBATCH --cpus-per-task=32              # Recommend 8 per GPU

backups/scripts/hf_model.py

@@ -15,9 +15,9 @@ def upload():
 
         try:
             if token is not None:
-                upload_folder(repo_id=args.repo_id, folder_path=args.folder_path, ignore_patterns=ignore_patterns, token=token)
+                upload_folder(repo_id=args.repo_id, folder_path=args.folder_path, ignore_patterns=ignore_patterns, path_in_repo=args.path_in_repo, token=token)
             else:
-                upload_folder(repo_id=args.repo_id, folder_path=args.folder_path, ignore_patterns=ignore_patterns)
+                upload_folder(repo_id=args.repo_id, folder_path=args.folder_path, ignore_patterns=ignore_patterns, path_in_repo=args.path_in_repo)
             table = Table(title=None, show_header=False, box=box.MINIMAL, title_style=style.Style(bold=True))
             table.add_row(f"Model id {args.repo_id}", str(args.folder_path))
             CONSOLE.print(Panel(table, title="[bold][green]:tada: Upload completed DO NOT forget specify the model id in methods! :tada:[/bold]", expand=False))
@@ -92,6 +92,7 @@ if __name__ == "__main__":
     parser.add_argument("--file_path", type=str, default=None, required=False)
     parser.add_argument("--save_path", type=str, default=None, required=False)
     parser.add_argument("--token", type=str, default=None, required=False)
+    parser.add_argument("--path_in_repo", type=str, default=None, required=False)
     args = parser.parse_args()
 
     token = args.token or os.getenv("hf_token", None)

backups/scripts/run_eval.sh

@@ -1,5 +1,17 @@
 #! /bin/bash
 
+# network
+source /baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/clash.sh
+bash /baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/clash-for-linux-backup/start.sh
+proxy_on
+
+# env
+source /baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/anaconda3/etc/profile.d/conda.sh
+conda config --append envs_dirs /baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/.conda/envs
+conda activate traj
+
+cd /baai-cwm-1/baai_cwm_ml/algorithm/xiuyu.yang/work/dev4/
+
 # env
 export REQUESTS_CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"
 export HTTPS_PROXY="https://192.168.0.10:443/"
@@ -11,10 +23,12 @@ export WANDB=1
 DEVICES=$1
 CONFIG='configs/ours_long_term.yaml'
 # CKPT_PATH='output/scalable_smart_long/last.ckpt'
-CKPT_PATH='output2/seq_10_150_3_3_encode_occ_separate_offsets_bs8_128_no_seqindex_long/last.ckpt'
+# CKPT_PATH='output2/seq_10_150_3_3_encode_occ_separate_offsets_bs8_128_no_seqindex_long/last.ckpt'
+CKPT_PATH='output2/seq_10_150_3_3_encode_occ_separate_offsets_bs8_128_no_seqindex_long_lastvalid_ablation_grid/last.ckpt'
 
 # run
 PYTHONPATH=".":$PYTHONPATH python3 run.py \
                             --devices $DEVICES \
                             --config $CONFIG \
                             --ckpt_path $CKPT_PATH ${@:2}
+

epoch=31.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e96741caea6c6b048b8dfa1b60ac3ad39435d42cbf06bdbd6fb84b25e1e2df8
+size 135738582

last.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25cfa41243cbea3b0d15c1414b767ec564ecca2656f0f2d78b13180d00b5c2da
-size 116282362
+oid sha256:98509b491480103452e868729cbaba73eb96c07ba60bb344d9d851e4789aafaf
+size 135738582