diff --git a/checkpoints/000100/pretrained_model/config.json b/checkpoints/000100/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..726734e1db80d6873b4ee49a62efd56d5767d628 --- /dev/null +++ b/checkpoints/000100/pretrained_model/config.json @@ -0,0 +1,72 @@ +{ + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 +} \ No newline at end of file diff --git a/checkpoints/000100/pretrained_model/model.safetensors b/checkpoints/000100/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cf0374d14bf26974cbd81c1cddbf8b4827f08096 --- /dev/null +++ b/checkpoints/000100/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1caa4a8ce1a72347a568aee5be82b8a9de987b56d8a76bde783310d8daf6f21e +size 7536022544 diff --git a/checkpoints/000100/pretrained_model/train_config.json b/checkpoints/000100/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0464eee65fe050ad04c44016031ffd41fd7ddf --- /dev/null +++ b/checkpoints/000100/pretrained_model/train_config.json @@ -0,0 +1,184 @@ +{ + "dataset": { + "repo_id": "maelic/hackathon7", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 + }, + "output_dir": "/scratch/train/pi_shity_version", + "job_name": "pi0", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 20, + "steps": 100000, + "eval_freq": 100, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 100, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2.5e-05, + "weight_decay": 1e-10, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2.5e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "pi0_mistral_hackathon", + "entity": null, + "notes": null, + "run_id": "shity_version", + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/000100/training_state/optimizer_param_groups.json b/checkpoints/000100/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..6d844a9a1a2210332cc4717701995ee4bc8e94de --- /dev/null +++ b/checkpoints/000100/training_state/optimizer_param_groups.json @@ -0,0 +1,803 @@ +[ + { + "lr": 2.5224775224775203e-06, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 1e-10, + "amsgrad": false, + "foreach": null, + "maximize": false, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 2.5e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + 512, + 513, + 514, + 515, + 516, + 517, + 518, + 519, + 520, + 521, + 522, + 523, + 524, + 525, + 526, + 527, + 528, + 529, + 530, + 531, + 532, + 533, + 534, + 535, + 536, + 537, + 538, + 539, + 540, + 541, + 542, + 543, + 544, + 545, + 546, + 547, + 548, + 549, + 550, + 551, + 552, + 553, + 554, + 555, + 556, + 557, + 558, + 559, + 560, + 561, + 562, + 563, + 564, + 565, + 566, + 567, + 568, + 569, + 570, + 571, + 572, + 573, + 574, + 575, + 576, + 577, + 578, + 579, + 580, + 581, + 582, + 583, + 584, + 585, + 586, + 587, + 588, + 589, + 590, + 591, + 592, + 593, + 594, + 595, + 596, + 597, + 598, + 599, + 600, + 601, + 602, + 603, + 604, + 605, + 606, + 607, + 608, + 609, + 610, + 611, + 612, + 613, + 614, + 615, + 616, + 617, + 618, + 619, + 620, + 621, + 622, + 623, + 624, + 625, + 626, + 627, + 628, + 629, + 630, + 631, + 632, + 633, + 634, + 635, + 636, + 637, + 638, + 639, + 640, + 641, + 642, + 643, + 644, + 645, + 646, + 647, + 648, + 649, + 650, + 651, + 652, + 653, + 654, + 655, + 656, + 657, + 658, + 659, + 660, + 661, + 662, + 663, + 664, + 665, + 666, + 667, + 668, + 669, + 670, + 671, + 672, + 673, + 674, + 675, + 676, + 677, + 678, + 679, + 680, + 681, + 682, + 683, + 684, + 685, + 686, + 687, + 688, + 689, + 690, + 691, + 692, + 693, + 694, + 695, + 696, + 697, + 698, + 699, + 700, + 701, + 702, + 703, + 704, + 705, + 706, + 707, + 708, + 709, + 710, + 711, + 712, + 713, + 714, + 715, + 716, + 717, + 718, + 719, + 720, + 721, + 722, + 723, + 724, + 725, + 726, + 727, + 728, + 729, + 730, + 731, + 732, + 733, + 734, + 735, + 736, + 737, + 738, + 739, + 740, + 741, + 742, + 743, + 744, + 745, + 746, + 747, + 748, + 749, + 750, + 751, + 752, + 753, + 754, + 755, + 756, + 757, + 758, + 759, + 760, + 761, + 762, + 763, + 764, + 765, + 766, + 767, + 768, + 769, + 770, + 771, + 772, + 773, + 774, + 775, + 776, + 777, + 778, + 779, + 780, + 781, + 782 + ] + } +] \ No newline at end of file diff --git a/checkpoints/000100/training_state/optimizer_state.safetensors b/checkpoints/000100/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8b4712087256380c44c2dfca56cfe2255c4f47d6 --- /dev/null +++ b/checkpoints/000100/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d5f93cd450bd32b7259e359a31b76b796830dcfa4edb6e15c94be9e35c4f540 +size 10896063516 diff --git a/checkpoints/000100/training_state/rng_state.safetensors b/checkpoints/000100/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7699eb0662d4fa39536a3559128dc979ba28d055 --- /dev/null +++ b/checkpoints/000100/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca8021aba17e321480ce6967b04e6ba9bc485bd9b6ab1e626759e196a9c20e73 +size 15708 diff --git a/checkpoints/000100/training_state/scheduler_state.json b/checkpoints/000100/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7e91e8acba3c2d4b86435fcc624d631d88a4d54d --- /dev/null +++ b/checkpoints/000100/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2.5e-05 + ], + "last_epoch": 100, + "verbose": false, + "_step_count": 101, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.5224775224775203e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/000100/training_state/training_step.json b/checkpoints/000100/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..09d55d5e2f831720b1916b01ffa45d2b6e68a654 --- /dev/null +++ b/checkpoints/000100/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 100 +} \ No newline at end of file diff --git a/checkpoints/000200/pretrained_model/config.json b/checkpoints/000200/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..726734e1db80d6873b4ee49a62efd56d5767d628 --- /dev/null +++ b/checkpoints/000200/pretrained_model/config.json @@ -0,0 +1,72 @@ +{ + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 +} \ No newline at end of file diff --git a/checkpoints/000200/pretrained_model/model.safetensors b/checkpoints/000200/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5ef66b88200d04e02a4fbbbd966a85384abd74be --- /dev/null +++ b/checkpoints/000200/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba1226f37c19f01ce308a286bd5daf7ba1760bd1030913455df96c63c152d069 +size 7536022544 diff --git a/checkpoints/000200/pretrained_model/train_config.json b/checkpoints/000200/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0464eee65fe050ad04c44016031ffd41fd7ddf --- /dev/null +++ b/checkpoints/000200/pretrained_model/train_config.json @@ -0,0 +1,184 @@ +{ + "dataset": { + "repo_id": "maelic/hackathon7", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 + }, + "output_dir": "/scratch/train/pi_shity_version", + "job_name": "pi0", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 20, + "steps": 100000, + "eval_freq": 100, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 100, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2.5e-05, + "weight_decay": 1e-10, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2.5e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "pi0_mistral_hackathon", + "entity": null, + "notes": null, + "run_id": "shity_version", + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/000200/training_state/optimizer_param_groups.json b/checkpoints/000200/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..de7ef68990f543f13cf9fb3f85ffbda4e2bc6d2e --- /dev/null +++ b/checkpoints/000200/training_state/optimizer_param_groups.json @@ -0,0 +1,803 @@ +[ + { + "lr": 5.0199800199800195e-06, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 1e-10, + "amsgrad": false, + "foreach": null, + "maximize": false, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 2.5e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + 512, + 513, + 514, + 515, + 516, + 517, + 518, + 519, + 520, + 521, + 522, + 523, + 524, + 525, + 526, + 527, + 528, + 529, + 530, + 531, + 532, + 533, + 534, + 535, + 536, + 537, + 538, + 539, + 540, + 541, + 542, + 543, + 544, + 545, + 546, + 547, + 548, + 549, + 550, + 551, + 552, + 553, + 554, + 555, + 556, + 557, + 558, + 559, + 560, + 561, + 562, + 563, + 564, + 565, + 566, + 567, + 568, + 569, + 570, + 571, + 572, + 573, + 574, + 575, + 576, + 577, + 578, + 579, + 580, + 581, + 582, + 583, + 584, + 585, + 586, + 587, + 588, + 589, + 590, + 591, + 592, + 593, + 594, + 595, + 596, + 597, + 598, + 599, + 600, + 601, + 602, + 603, + 604, + 605, + 606, + 607, + 608, + 609, + 610, + 611, + 612, + 613, + 614, + 615, + 616, + 617, + 618, + 619, + 620, + 621, + 622, + 623, + 624, + 625, + 626, + 627, + 628, + 629, + 630, + 631, + 632, + 633, + 634, + 635, + 636, + 637, + 638, + 639, + 640, + 641, + 642, + 643, + 644, + 645, + 646, + 647, + 648, + 649, + 650, + 651, + 652, + 653, + 654, + 655, + 656, + 657, + 658, + 659, + 660, + 661, + 662, + 663, + 664, + 665, + 666, + 667, + 668, + 669, + 670, + 671, + 672, + 673, + 674, + 675, + 676, + 677, + 678, + 679, + 680, + 681, + 682, + 683, + 684, + 685, + 686, + 687, + 688, + 689, + 690, + 691, + 692, + 693, + 694, + 695, + 696, + 697, + 698, + 699, + 700, + 701, + 702, + 703, + 704, + 705, + 706, + 707, + 708, + 709, + 710, + 711, + 712, + 713, + 714, + 715, + 716, + 717, + 718, + 719, + 720, + 721, + 722, + 723, + 724, + 725, + 726, + 727, + 728, + 729, + 730, + 731, + 732, + 733, + 734, + 735, + 736, + 737, + 738, + 739, + 740, + 741, + 742, + 743, + 744, + 745, + 746, + 747, + 748, + 749, + 750, + 751, + 752, + 753, + 754, + 755, + 756, + 757, + 758, + 759, + 760, + 761, + 762, + 763, + 764, + 765, + 766, + 767, + 768, + 769, + 770, + 771, + 772, + 773, + 774, + 775, + 776, + 777, + 778, + 779, + 780, + 781, + 782 + ] + } +] \ No newline at end of file diff --git a/checkpoints/000200/training_state/optimizer_state.safetensors b/checkpoints/000200/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..30e1a587fb37072f09743fecdbbc79c60494deef --- /dev/null +++ b/checkpoints/000200/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdd7946065073cfc15fcadfdb18a6c52823ace248e63f04898e07bb4a504de19 +size 10896063516 diff --git a/checkpoints/000200/training_state/rng_state.safetensors b/checkpoints/000200/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c1b5de4b2443caadfe253301beabcb221503e2ed --- /dev/null +++ b/checkpoints/000200/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:184b214b2feea731bbad0502189e04546bee7137c89d7cf3edb66f6e065222ce +size 15708 diff --git a/checkpoints/000200/training_state/scheduler_state.json b/checkpoints/000200/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d6cbb9a1845b3c9e5297d7bb38cf81a0fa1387b2 --- /dev/null +++ b/checkpoints/000200/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2.5e-05 + ], + "last_epoch": 200, + "verbose": false, + "_step_count": 201, + "_get_lr_called_within_step": false, + "_last_lr": [ + 5.0199800199800195e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/000200/training_state/training_step.json b/checkpoints/000200/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..2d089762162bad11970028a428dbb187a42839df --- /dev/null +++ b/checkpoints/000200/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 200 +} \ No newline at end of file diff --git a/checkpoints/000300/pretrained_model/config.json b/checkpoints/000300/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..726734e1db80d6873b4ee49a62efd56d5767d628 --- /dev/null +++ b/checkpoints/000300/pretrained_model/config.json @@ -0,0 +1,72 @@ +{ + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 +} \ No newline at end of file diff --git a/checkpoints/000300/pretrained_model/model.safetensors b/checkpoints/000300/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7917b056919bb472c0c68c8124cd16f7d9a94b2a --- /dev/null +++ b/checkpoints/000300/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9c8be964aa757b69ece2c4c67b5972e36d6b0bd1dacf0ee3af709c0baa71846 +size 7536022544 diff --git a/checkpoints/000300/pretrained_model/train_config.json b/checkpoints/000300/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0464eee65fe050ad04c44016031ffd41fd7ddf --- /dev/null +++ b/checkpoints/000300/pretrained_model/train_config.json @@ -0,0 +1,184 @@ +{ + "dataset": { + "repo_id": "maelic/hackathon7", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 + }, + "output_dir": "/scratch/train/pi_shity_version", + "job_name": "pi0", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 20, + "steps": 100000, + "eval_freq": 100, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 100, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2.5e-05, + "weight_decay": 1e-10, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2.5e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "pi0_mistral_hackathon", + "entity": null, + "notes": null, + "run_id": "shity_version", + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/000300/training_state/optimizer_param_groups.json b/checkpoints/000300/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..069cbfc483fe828dbed3a35d8756d7f63fb5973d --- /dev/null +++ b/checkpoints/000300/training_state/optimizer_param_groups.json @@ -0,0 +1,803 @@ +[ + { + "lr": 7.517482517482519e-06, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 1e-10, + "amsgrad": false, + "foreach": null, + "maximize": false, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 2.5e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + 512, + 513, + 514, + 515, + 516, + 517, + 518, + 519, + 520, + 521, + 522, + 523, + 524, + 525, + 526, + 527, + 528, + 529, + 530, + 531, + 532, + 533, + 534, + 535, + 536, + 537, + 538, + 539, + 540, + 541, + 542, + 543, + 544, + 545, + 546, + 547, + 548, + 549, + 550, + 551, + 552, + 553, + 554, + 555, + 556, + 557, + 558, + 559, + 560, + 561, + 562, + 563, + 564, + 565, + 566, + 567, + 568, + 569, + 570, + 571, + 572, + 573, + 574, + 575, + 576, + 577, + 578, + 579, + 580, + 581, + 582, + 583, + 584, + 585, + 586, + 587, + 588, + 589, + 590, + 591, + 592, + 593, + 594, + 595, + 596, + 597, + 598, + 599, + 600, + 601, + 602, + 603, + 604, + 605, + 606, + 607, + 608, + 609, + 610, + 611, + 612, + 613, + 614, + 615, + 616, + 617, + 618, + 619, + 620, + 621, + 622, + 623, + 624, + 625, + 626, + 627, + 628, + 629, + 630, + 631, + 632, + 633, + 634, + 635, + 636, + 637, + 638, + 639, + 640, + 641, + 642, + 643, + 644, + 645, + 646, + 647, + 648, + 649, + 650, + 651, + 652, + 653, + 654, + 655, + 656, + 657, + 658, + 659, + 660, + 661, + 662, + 663, + 664, + 665, + 666, + 667, + 668, + 669, + 670, + 671, + 672, + 673, + 674, + 675, + 676, + 677, + 678, + 679, + 680, + 681, + 682, + 683, + 684, + 685, + 686, + 687, + 688, + 689, + 690, + 691, + 692, + 693, + 694, + 695, + 696, + 697, + 698, + 699, + 700, + 701, + 702, + 703, + 704, + 705, + 706, + 707, + 708, + 709, + 710, + 711, + 712, + 713, + 714, + 715, + 716, + 717, + 718, + 719, + 720, + 721, + 722, + 723, + 724, + 725, + 726, + 727, + 728, + 729, + 730, + 731, + 732, + 733, + 734, + 735, + 736, + 737, + 738, + 739, + 740, + 741, + 742, + 743, + 744, + 745, + 746, + 747, + 748, + 749, + 750, + 751, + 752, + 753, + 754, + 755, + 756, + 757, + 758, + 759, + 760, + 761, + 762, + 763, + 764, + 765, + 766, + 767, + 768, + 769, + 770, + 771, + 772, + 773, + 774, + 775, + 776, + 777, + 778, + 779, + 780, + 781, + 782 + ] + } +] \ No newline at end of file diff --git a/checkpoints/000300/training_state/optimizer_state.safetensors b/checkpoints/000300/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..768b51711adfc3035b86d5c37729d4247c5fe541 --- /dev/null +++ b/checkpoints/000300/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5771d887785d5a9a9efae7b05f9f98c8bc6d28424ca17ee67b661fbff4cea8e +size 10896063516 diff --git a/checkpoints/000300/training_state/rng_state.safetensors b/checkpoints/000300/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..160d187290e4cf069f256886a6ef1284acf16a3e --- /dev/null +++ b/checkpoints/000300/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5aaa23448022e37fd9f49f7682563e654bfc12e356201d47e8d5e8a07904a6b8 +size 15708 diff --git a/checkpoints/000300/training_state/scheduler_state.json b/checkpoints/000300/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e62d1ef32ee9894cdce55ad1507581924fa53aa2 --- /dev/null +++ b/checkpoints/000300/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2.5e-05 + ], + "last_epoch": 300, + "verbose": false, + "_step_count": 301, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.517482517482519e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/000300/training_state/training_step.json b/checkpoints/000300/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..0533f3e8562df02261dad4b8762034a8a78504dc --- /dev/null +++ b/checkpoints/000300/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 300 +} \ No newline at end of file diff --git a/checkpoints/000400/pretrained_model/config.json b/checkpoints/000400/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..726734e1db80d6873b4ee49a62efd56d5767d628 --- /dev/null +++ b/checkpoints/000400/pretrained_model/config.json @@ -0,0 +1,72 @@ +{ + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 +} \ No newline at end of file diff --git a/checkpoints/000400/pretrained_model/model.safetensors b/checkpoints/000400/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6169048c3e146315fb600b0424d18b672c34886c --- /dev/null +++ b/checkpoints/000400/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce98db53ec22353c410f32ec1e332eb008aa222ab73292c4697b026927e55d44 +size 7536022544 diff --git a/checkpoints/000400/pretrained_model/train_config.json b/checkpoints/000400/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0464eee65fe050ad04c44016031ffd41fd7ddf --- /dev/null +++ b/checkpoints/000400/pretrained_model/train_config.json @@ -0,0 +1,184 @@ +{ + "dataset": { + "repo_id": "maelic/hackathon7", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 + }, + "output_dir": "/scratch/train/pi_shity_version", + "job_name": "pi0", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 20, + "steps": 100000, + "eval_freq": 100, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 100, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2.5e-05, + "weight_decay": 1e-10, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2.5e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "pi0_mistral_hackathon", + "entity": null, + "notes": null, + "run_id": "shity_version", + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/000400/training_state/optimizer_param_groups.json b/checkpoints/000400/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..f892bca0487162fe3347f1f9884fee8b29107db1 --- /dev/null +++ b/checkpoints/000400/training_state/optimizer_param_groups.json @@ -0,0 +1,803 @@ +[ + { + "lr": 1.0014985014985015e-05, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 1e-10, + "amsgrad": false, + "foreach": null, + "maximize": false, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 2.5e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + 512, + 513, + 514, + 515, + 516, + 517, + 518, + 519, + 520, + 521, + 522, + 523, + 524, + 525, + 526, + 527, + 528, + 529, + 530, + 531, + 532, + 533, + 534, + 535, + 536, + 537, + 538, + 539, + 540, + 541, + 542, + 543, + 544, + 545, + 546, + 547, + 548, + 549, + 550, + 551, + 552, + 553, + 554, + 555, + 556, + 557, + 558, + 559, + 560, + 561, + 562, + 563, + 564, + 565, + 566, + 567, + 568, + 569, + 570, + 571, + 572, + 573, + 574, + 575, + 576, + 577, + 578, + 579, + 580, + 581, + 582, + 583, + 584, + 585, + 586, + 587, + 588, + 589, + 590, + 591, + 592, + 593, + 594, + 595, + 596, + 597, + 598, + 599, + 600, + 601, + 602, + 603, + 604, + 605, + 606, + 607, + 608, + 609, + 610, + 611, + 612, + 613, + 614, + 615, + 616, + 617, + 618, + 619, + 620, + 621, + 622, + 623, + 624, + 625, + 626, + 627, + 628, + 629, + 630, + 631, + 632, + 633, + 634, + 635, + 636, + 637, + 638, + 639, + 640, + 641, + 642, + 643, + 644, + 645, + 646, + 647, + 648, + 649, + 650, + 651, + 652, + 653, + 654, + 655, + 656, + 657, + 658, + 659, + 660, + 661, + 662, + 663, + 664, + 665, + 666, + 667, + 668, + 669, + 670, + 671, + 672, + 673, + 674, + 675, + 676, + 677, + 678, + 679, + 680, + 681, + 682, + 683, + 684, + 685, + 686, + 687, + 688, + 689, + 690, + 691, + 692, + 693, + 694, + 695, + 696, + 697, + 698, + 699, + 700, + 701, + 702, + 703, + 704, + 705, + 706, + 707, + 708, + 709, + 710, + 711, + 712, + 713, + 714, + 715, + 716, + 717, + 718, + 719, + 720, + 721, + 722, + 723, + 724, + 725, + 726, + 727, + 728, + 729, + 730, + 731, + 732, + 733, + 734, + 735, + 736, + 737, + 738, + 739, + 740, + 741, + 742, + 743, + 744, + 745, + 746, + 747, + 748, + 749, + 750, + 751, + 752, + 753, + 754, + 755, + 756, + 757, + 758, + 759, + 760, + 761, + 762, + 763, + 764, + 765, + 766, + 767, + 768, + 769, + 770, + 771, + 772, + 773, + 774, + 775, + 776, + 777, + 778, + 779, + 780, + 781, + 782 + ] + } +] \ No newline at end of file diff --git a/checkpoints/000400/training_state/optimizer_state.safetensors b/checkpoints/000400/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1b2d6df1d39f496681de75ba71d43ab4127325fa --- /dev/null +++ b/checkpoints/000400/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53cd76132357b389f0deb59640ffcdd70879e9c82bfb0de11f2669eaa5f68993 +size 10896063516 diff --git a/checkpoints/000400/training_state/rng_state.safetensors b/checkpoints/000400/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e65a3391f763287c018a3e2908ac4ead9819de35 --- /dev/null +++ b/checkpoints/000400/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b56e833e6eea05811578c3ab02d4aa80f853a9a2aa2921cd1638912496d0193e +size 15708 diff --git a/checkpoints/000400/training_state/scheduler_state.json b/checkpoints/000400/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4f2fcebe79745f15ca951f8456d05173c2d36401 --- /dev/null +++ b/checkpoints/000400/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2.5e-05 + ], + "last_epoch": 400, + "verbose": false, + "_step_count": 401, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.0014985014985015e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/000400/training_state/training_step.json b/checkpoints/000400/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..709d48a709b1b2972a5b30cf09b63497c3439d6b --- /dev/null +++ b/checkpoints/000400/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 400 +} \ No newline at end of file diff --git a/checkpoints/000500/pretrained_model/config.json b/checkpoints/000500/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..726734e1db80d6873b4ee49a62efd56d5767d628 --- /dev/null +++ b/checkpoints/000500/pretrained_model/config.json @@ -0,0 +1,72 @@ +{ + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 +} \ No newline at end of file diff --git a/checkpoints/000500/pretrained_model/model.safetensors b/checkpoints/000500/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..37e59e22abfdf0afea6427a8b14783a9d6bf61cf --- /dev/null +++ b/checkpoints/000500/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5234e087c1cb0af3d9e7960a6766315f21d0344a5f7212e5aaaa5e7684aa8c8a +size 7536022544 diff --git a/checkpoints/000500/pretrained_model/train_config.json b/checkpoints/000500/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0464eee65fe050ad04c44016031ffd41fd7ddf --- /dev/null +++ b/checkpoints/000500/pretrained_model/train_config.json @@ -0,0 +1,184 @@ +{ + "dataset": { + "repo_id": "maelic/hackathon7", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 + }, + "output_dir": "/scratch/train/pi_shity_version", + "job_name": "pi0", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 20, + "steps": 100000, + "eval_freq": 100, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 100, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2.5e-05, + "weight_decay": 1e-10, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2.5e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "pi0_mistral_hackathon", + "entity": null, + "notes": null, + "run_id": "shity_version", + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/000500/training_state/optimizer_param_groups.json b/checkpoints/000500/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..72e274ea151ad1d83572febe7ec88dfe0cc3bfb2 --- /dev/null +++ b/checkpoints/000500/training_state/optimizer_param_groups.json @@ -0,0 +1,803 @@ +[ + { + "lr": 1.2512487512487514e-05, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 1e-10, + "amsgrad": false, + "foreach": null, + "maximize": false, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 2.5e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + 512, + 513, + 514, + 515, + 516, + 517, + 518, + 519, + 520, + 521, + 522, + 523, + 524, + 525, + 526, + 527, + 528, + 529, + 530, + 531, + 532, + 533, + 534, + 535, + 536, + 537, + 538, + 539, + 540, + 541, + 542, + 543, + 544, + 545, + 546, + 547, + 548, + 549, + 550, + 551, + 552, + 553, + 554, + 555, + 556, + 557, + 558, + 559, + 560, + 561, + 562, + 563, + 564, + 565, + 566, + 567, + 568, + 569, + 570, + 571, + 572, + 573, + 574, + 575, + 576, + 577, + 578, + 579, + 580, + 581, + 582, + 583, + 584, + 585, + 586, + 587, + 588, + 589, + 590, + 591, + 592, + 593, + 594, + 595, + 596, + 597, + 598, + 599, + 600, + 601, + 602, + 603, + 604, + 605, + 606, + 607, + 608, + 609, + 610, + 611, + 612, + 613, + 614, + 615, + 616, + 617, + 618, + 619, + 620, + 621, + 622, + 623, + 624, + 625, + 626, + 627, + 628, + 629, + 630, + 631, + 632, + 633, + 634, + 635, + 636, + 637, + 638, + 639, + 640, + 641, + 642, + 643, + 644, + 645, + 646, + 647, + 648, + 649, + 650, + 651, + 652, + 653, + 654, + 655, + 656, + 657, + 658, + 659, + 660, + 661, + 662, + 663, + 664, + 665, + 666, + 667, + 668, + 669, + 670, + 671, + 672, + 673, + 674, + 675, + 676, + 677, + 678, + 679, + 680, + 681, + 682, + 683, + 684, + 685, + 686, + 687, + 688, + 689, + 690, + 691, + 692, + 693, + 694, + 695, + 696, + 697, + 698, + 699, + 700, + 701, + 702, + 703, + 704, + 705, + 706, + 707, + 708, + 709, + 710, + 711, + 712, + 713, + 714, + 715, + 716, + 717, + 718, + 719, + 720, + 721, + 722, + 723, + 724, + 725, + 726, + 727, + 728, + 729, + 730, + 731, + 732, + 733, + 734, + 735, + 736, + 737, + 738, + 739, + 740, + 741, + 742, + 743, + 744, + 745, + 746, + 747, + 748, + 749, + 750, + 751, + 752, + 753, + 754, + 755, + 756, + 757, + 758, + 759, + 760, + 761, + 762, + 763, + 764, + 765, + 766, + 767, + 768, + 769, + 770, + 771, + 772, + 773, + 774, + 775, + 776, + 777, + 778, + 779, + 780, + 781, + 782 + ] + } +] \ No newline at end of file diff --git a/checkpoints/000500/training_state/optimizer_state.safetensors b/checkpoints/000500/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aebb47cfc9201096835dafbe85c548e9f8b47315 --- /dev/null +++ b/checkpoints/000500/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2818383622ba294aa20d4dea9301559ea08a137ca79ec835802d1122afaf7c61 +size 10896063516 diff --git a/checkpoints/000500/training_state/rng_state.safetensors b/checkpoints/000500/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e7036990116eb3d92fd6402449ffd6dfdef6c66c --- /dev/null +++ b/checkpoints/000500/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:908dcf320b9f7841453ea8d3ee5603979dd20285f641c0fe37229eb9b3c41074 +size 15708 diff --git a/checkpoints/000500/training_state/scheduler_state.json b/checkpoints/000500/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e56e97a6a743a18326a627f9d1cb569fdbd64843 --- /dev/null +++ b/checkpoints/000500/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2.5e-05 + ], + "last_epoch": 500, + "verbose": false, + "_step_count": 501, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.2512487512487514e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/000500/training_state/training_step.json b/checkpoints/000500/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8df625539d7e10ddfbd1e2ac0daf2fdd64ff5c4b --- /dev/null +++ b/checkpoints/000500/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 500 +} \ No newline at end of file diff --git a/checkpoints/000600/pretrained_model/config.json b/checkpoints/000600/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..726734e1db80d6873b4ee49a62efd56d5767d628 --- /dev/null +++ b/checkpoints/000600/pretrained_model/config.json @@ -0,0 +1,72 @@ +{ + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 +} \ No newline at end of file diff --git a/checkpoints/000600/pretrained_model/model.safetensors b/checkpoints/000600/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..70166bb65b1ba0b71eb04f23594716f2b7052b1a --- /dev/null +++ b/checkpoints/000600/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e4e6e3f950fdde0460bdd66685b7e7b43cd38d01258624d7252f1bd26d5c54d +size 7536022544 diff --git a/checkpoints/000600/pretrained_model/train_config.json b/checkpoints/000600/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0464eee65fe050ad04c44016031ffd41fd7ddf --- /dev/null +++ b/checkpoints/000600/pretrained_model/train_config.json @@ -0,0 +1,184 @@ +{ + "dataset": { + "repo_id": "maelic/hackathon7", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "pi0", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.gripper": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.webcam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 224, + 224 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "proj_width": 1024, + "num_steps": 10, + "use_cache": true, + "attention_implementation": "eager", + "freeze_vision_encoder": true, + "train_expert_only": false, + "train_state_proj": true, + "optimizer_lr": 2.5e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-10, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06 + }, + "output_dir": "/scratch/train/pi_shity_version", + "job_name": "pi0", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 20, + "steps": 100000, + "eval_freq": 100, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 100, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2.5e-05, + "weight_decay": 1e-10, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2.5e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "pi0_mistral_hackathon", + "entity": null, + "notes": null, + "run_id": "shity_version", + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/000600/training_state/rng_state.safetensors b/checkpoints/000600/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4b0d612965e7ffe8240d0eee0b3527e13ae6dfba --- /dev/null +++ b/checkpoints/000600/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:947a8b9f49f0bd12461173ac9c0ddfa6a97daf7a1f13fc015d1e1148ce86ba4b +size 15708 diff --git a/checkpoints/000600/training_state/training_step.json b/checkpoints/000600/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..d28829bb88ed269920ca872989632b0c97c31164 --- /dev/null +++ b/checkpoints/000600/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 600 +} \ No newline at end of file diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..89eb1760a519725769ed2201015e9ecb3fec3900 --- /dev/null +++ b/wandb/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-04-12T22:01:36.520670877Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/scratch/train/pi_shity_version/wandb/run-20250412_220136-shity_version/logs/debug-core.log"} +{"time":"2025-04-12T22:01:36.769914227Z","level":"INFO","msg":"created new stream","id":"shity_version"} +{"time":"2025-04-12T22:01:36.769954097Z","level":"INFO","msg":"stream: started","id":"shity_version"} +{"time":"2025-04-12T22:01:36.769978883Z","level":"INFO","msg":"writer: Do: started","stream_id":"shity_version"} +{"time":"2025-04-12T22:01:36.769995438Z","level":"INFO","msg":"handler: started","stream_id":"shity_version"} +{"time":"2025-04-12T22:01:36.770025753Z","level":"INFO","msg":"sender: started","stream_id":"shity_version"} +{"time":"2025-04-12T22:01:37.196484495Z","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/debug.log b/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d240febd271d7743d425683659882bb0ea7e8978 --- /dev/null +++ b/wandb/debug.log @@ -0,0 +1,23 @@ +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_setup.py:_flush():67] Configure stats pid to 27554 +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_setup.py:_flush():67] Loading settings from /root/lerobot/wandb/settings +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /scratch/train/pi_shity_version/wandb/run-20250412_220136-shity_version/logs/debug.log +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /scratch/train/pi_shity_version/wandb/run-20250412_220136-shity_version/logs/debug-internal.log +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_init.py:init():781] calling init triggers +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'maelic/hackathon7', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'pi0', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {'observation.state': {'type': , 'shape': [6]}, 'observation.images.gripper': {'type': , 'shape': [3, 480, 640]}, 'observation.images.webcam': {'type': , 'shape': [3, 480, 640]}}, 'output_features': {'action': {'type': , 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 50, 'n_action_steps': 50, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [224, 224], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'proj_width': 1024, 'num_steps': 10, 'use_cache': True, 'attention_implementation': 'eager', 'freeze_vision_encoder': True, 'train_expert_only': False, 'train_state_proj': True, 'optimizer_lr': 2.5e-05, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-10, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06}, 'output_dir': '/scratch/train/pi_shity_version', 'job_name': 'pi0', 'resume': True, 'seed': 1000, 'num_workers': 4, 'batch_size': 20, 'steps': 100000, 'eval_freq': 100, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 100, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 2.5e-05, 'weight_decay': 1e-10, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 2.5e-05, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'pi0_mistral_hackathon', 'entity': None, 'notes': None, 'run_id': 'shity_version', 'mode': None}, '_wandb': {}} +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_init.py:init():809] starting backend +2025-04-12 22:01:36,516 INFO MainThread:27554 [wandb_init.py:init():813] sending inform_init request +2025-04-12 22:01:36,519 INFO MainThread:27554 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-12 22:01:36,519 INFO MainThread:27554 [wandb_init.py:init():823] backend started and connected +2025-04-12 22:01:36,520 INFO MainThread:27554 [wandb_init.py:init():915] updated telemetry +2025-04-12 22:01:36,524 INFO MainThread:27554 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-04-12 22:01:37,193 INFO MainThread:27554 [wandb_init.py:init():1009] run resumed +2025-04-12 22:01:37,195 INFO MainThread:27554 [wandb_init.py:init():1014] starting run threads in backend +2025-04-12 22:01:37,293 INFO MainThread:27554 [wandb_run.py:_console_start():2454] atexit reg +2025-04-12 22:01:37,294 INFO MainThread:27554 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-04-12 22:01:37,294 INFO MainThread:27554 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-04-12 22:01:37,294 INFO MainThread:27554 [wandb_run.py:_redirect():2394] Redirects installed. +2025-04-12 22:01:37,295 INFO MainThread:27554 [wandb_init.py:init():1056] run started, returning control to user process diff --git a/wandb/run-20250412_213619-shity_version/files/config.yaml b/wandb/run-20250412_213619-shity_version/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f602fd87398cb5dd9b6a4b5ee7ce141999516a44 --- /dev/null +++ b/wandb/run-20250412_213619-shity_version/files/config.yaml @@ -0,0 +1,175 @@ +_wandb: + value: + cli_version: 0.19.9 + m: [] + python_version: 3.10.13 + t: + "1": + - 1 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "3": + - 13 + - 14 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.10.13 + "5": 0.19.9 + "8": + - 5 + "12": 0.19.9 + "13": linux-x86_64 +batch_size: + value: 20 +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: maelic/hackathon7 + revision: null + root: null + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 100 +job_name: + value: pi0 +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.95 + eps: 1e-08 + grad_clip_norm: 10 + lr: 2.5e-05 + type: adamw + weight_decay: 1e-10 +output_dir: + value: /scratch/train/pi_shity_version +policy: + value: + adapt_to_pi_aloha: false + attention_implementation: eager + chunk_size: 50 + device: cuda + empty_cameras: 0 + freeze_vision_encoder: true + max_action_dim: 32 + max_state_dim: 32 + n_action_steps: 50 + n_obs_steps: 1 + normalization_mapping: + ACTION: MEAN_STD + STATE: MEAN_STD + VISUAL: IDENTITY + num_steps: 10 + optimizer_betas: + - 0.9 + - 0.95 + optimizer_eps: 1e-08 + optimizer_lr: 2.5e-05 + optimizer_weight_decay: 1e-10 + proj_width: 1024 + resize_imgs_with_padding: + - 224 + - 224 + scheduler_decay_lr: 2.5e-06 + scheduler_decay_steps: 30000 + scheduler_warmup_steps: 1000 + tokenizer_max_length: 48 + train_expert_only: false + train_state_proj: true + type: pi0 + use_amp: false + use_cache: true + use_delta_joint_actions_aloha: false +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 100 +scheduler: + value: + decay_lr: 2.5e-06 + num_decay_steps: 30000 + num_warmup_steps: 1000 + peak_lr: 2.5e-05 + type: cosine_decay_with_warmup +seed: + value: 1000 +steps: + value: 100000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: pi0_mistral_hackathon + run_id: shity_version diff --git a/wandb/run-20250412_213619-shity_version/files/output.log b/wandb/run-20250412_213619-shity_version/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..282e6f675ffe5cbccab6f63bba9ac967a1e22af8 --- /dev/null +++ b/wandb/run-20250412_213619-shity_version/files/output.log @@ -0,0 +1,75 @@ +Logs will be synced with wandb. +INFO 2025-04-12 21:36:19 ndb_utils.py:96 Track this run --> https://wandb.ai/helper2424-3commas/pi0_mistral_hackathon/runs/shity_version +INFO 2025-04-12 21:36:19 ts/train.py:127 Creating dataset +Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 30/30 [00:00<00:00, 205603.14it/s] +INFO 2025-04-12 21:36:20 ts/train.py:138 Creating policy +INFO 2025-04-12 21:37:06 ts/train.py:144 Creating optimizer and scheduler +INFO 2025-04-12 21:37:06 ts/train.py:156 Output dir: /scratch/train/pi_shity_version +INFO 2025-04-12 21:37:06 ts/train.py:159 cfg.steps=100000 (100K) +INFO 2025-04-12 21:37:06 ts/train.py:160 dataset.num_frames=11312 (11K) +INFO 2025-04-12 21:37:06 ts/train.py:161 dataset.num_episodes=30 +INFO 2025-04-12 21:37:06 ts/train.py:162 num_learnable_params=3088929824 (3B) +INFO 2025-04-12 21:37:06 ts/train.py:163 num_total_params=3501372212 (4B) +INFO 2025-04-12 21:37:06 ts/train.py:202 Start offline training on a fixed dataset +INFO 2025-04-12 21:38:23 ts/train.py:241 Checkpoint policy after step 100 +INFO 2025-04-12 21:40:25 ts/train.py:232 step:200 smpl:4K ep:11 epch:0.35 loss:0.065 grdn:1.385 lr:2.5e-06 updt_s:0.749 data_s:0.007 +WARNING 2025-04-12 21:40:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type is not handled by this wrapper. +WARNING 2025-04-12 21:40:25 db_utils.py:117 WandB logging of key "losses_after_in_ep_bound" was ignored as its type is not handled by this wrapper. +WARNING 2025-04-12 21:40:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type is not handled by this wrapper. +INFO 2025-04-12 21:40:25 ts/train.py:241 Checkpoint policy after step 200 +INFO 2025-04-12 21:42:23 ts/train.py:241 Checkpoint policy after step 300 +INFO 2025-04-12 21:44:25 ts/train.py:232 step:400 smpl:8K ep:21 epch:0.71 loss:0.033 grdn:0.748 lr:7.5e-06 updt_s:0.746 data_s:0.000 +WARNING 2025-04-12 21:44:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type is not handled by this wrapper. +WARNING 2025-04-12 21:44:25 db_utils.py:117 WandB logging of key "losses_after_in_ep_bound" was ignored as its type is not handled by this wrapper. +WARNING 2025-04-12 21:44:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type is not handled by this wrapper. +INFO 2025-04-12 21:44:25 ts/train.py:241 Checkpoint policy after step 400 +INFO 2025-04-12 21:46:23 ts/train.py:241 Checkpoint policy after step 500 +INFO 2025-04-12 21:48:23 ts/train.py:232 step:600 smpl:12K ep:32 epch:1.06 loss:0.022 grdn:0.634 lr:1.3e-05 updt_s:0.746 data_s:0.005 +WARNING 2025-04-12 21:48:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type is not handled by this wrapper. +WARNING 2025-04-12 21:48:23 db_utils.py:117 WandB logging of key "losses_after_in_ep_bound" was ignored as its type is not handled by this wrapper. +WARNING 2025-04-12 21:48:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type is not handled by this wrapper. +INFO 2025-04-12 21:48:23 ts/train.py:241 Checkpoint policy after step 600 +Traceback (most recent call last): + File "/root/lerobot/lerobot/scripts/train.py", line 288, in + train() + File "/root/lerobot/lerobot/configs/parser.py", line 227, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/root/lerobot/lerobot/scripts/train.py", line 243, in train + save_checkpoint(checkpoint_dir, step, cfg, policy, optimizer, lr_scheduler) + File "/root/lerobot/lerobot/common/utils/train_utils.py", line 102, in save_checkpoint + save_training_state(checkpoint_dir, step, optimizer, scheduler) + File "/root/lerobot/lerobot/common/utils/train_utils.py", line 127, in save_training_state + save_optimizer_state(optimizer, save_dir) + File "/root/lerobot/lerobot/common/optim/optimizers.py", line 101, in save_optimizer_state + save_file(flat_state, save_dir / OPTIMIZER_STATE) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/safetensors/torch.py", line 286, in save_file + serialize_file(_flatten(tensors), filename, metadata=metadata) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/safetensors/torch.py", line 496, in _flatten + return { + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/safetensors/torch.py", line 500, in + "data": _tobytes(v, k), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/safetensors/torch.py", line 422, in _tobytes + tensor = tensor.to("cpu") +KeyboardInterrupt +Traceback (most recent call last): + File "/root/lerobot/lerobot/scripts/train.py", line 288, in + train() + File "/root/lerobot/lerobot/configs/parser.py", line 227, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/root/lerobot/lerobot/scripts/train.py", line 243, in train + save_checkpoint(checkpoint_dir, step, cfg, policy, optimizer, lr_scheduler) + File "/root/lerobot/lerobot/common/utils/train_utils.py", line 102, in save_checkpoint + save_training_state(checkpoint_dir, step, optimizer, scheduler) + File "/root/lerobot/lerobot/common/utils/train_utils.py", line 127, in save_training_state + save_optimizer_state(optimizer, save_dir) + File "/root/lerobot/lerobot/common/optim/optimizers.py", line 101, in save_optimizer_state + save_file(flat_state, save_dir / OPTIMIZER_STATE) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/safetensors/torch.py", line 286, in save_file + serialize_file(_flatten(tensors), filename, metadata=metadata) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/safetensors/torch.py", line 496, in _flatten + return { + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/safetensors/torch.py", line 500, in + "data": _tobytes(v, k), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/safetensors/torch.py", line 422, in _tobytes + tensor = tensor.to("cpu") +KeyboardInterrupt diff --git a/wandb/run-20250412_213619-shity_version/files/requirements.txt b/wandb/run-20250412_213619-shity_version/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..19233a5893fbe938d47f546a80a53474123ed3f2 --- /dev/null +++ b/wandb/run-20250412_213619-shity_version/files/requirements.txt @@ -0,0 +1,245 @@ +tokenizers==0.21.1 +frozenlist==1.5.0 +numcodecs==0.13.1 +terminado==0.17.1 +Farama-Notifications==0.0.4 +labmaze==1.0.6 +Flask==3.1.0 +nvidia-cusparse-cu12==12.3.1.170 +jsonschema==4.23.0 +pycparser==2.21 +pycparser==2.22 +argon2-cffi==21.3.0 +multidict==6.4.3 +safetensors==0.5.3 +setuptools==75.8.0 +multiprocess==0.70.16 +urllib3==2.4.0 +urllib3==2.3.0 +gdown==5.2.0 +pytz==2025.2 +beautifulsoup4==4.12.3 +beautifulsoup4==4.13.3 +importlib_metadata==8.6.1 +babel==2.16.0 +cmake==4.0.0 +pyyaml-include==1.4.1 +asciitree==0.3.3 +h5py==3.13.0 +antlr4-python3-runtime==4.9.3 +tzdata==2025.2 +jupyterlab==4.3.4 +datasets==3.5.0 +mergedeep==1.3.4 +nvidia-cusparselt-cu12==0.6.2 +comm==0.2.1 +blinker==1.9.0 +referencing==0.30.2 +nvidia-cuda-cupti-cu12==12.4.127 +nbconvert==7.16.6 +iniconfig==2.1.0 +click==8.1.8 +nest-asyncio==1.6.0 +rerun-sdk==0.22.1 +wcwidth==0.2.5 +wcwidth==0.2.13 +wandb==0.19.9 +argon2-cffi-bindings==21.2.0 +nvidia-cusolver-cu12==11.6.1.9 +ptyprocess==0.7.0 +triton==3.2.0 +parso==0.8.4 +cffi==1.17.1 +executing==0.8.3 +soupsieve==2.6 +soupsieve==2.5 +lerobot==0.1.0 +nvidia-cuda-runtime-cu12==12.4.127 +python-dateutil==2.9.0.post0 +pure-eval==0.2.2 +aiosignal==1.3.2 +hf_transfer==0.1.9 +pandas==2.2.3 +dm-env==1.6 +pygame==2.6.1 +opencv-python-headless==4.11.0.86 +PyOpenGL==3.1.9 +matplotlib-inline==0.1.6 +requests==2.32.3 +propcache==0.3.1 +json5==0.9.25 +Pygments==2.15.1 +zipp==3.21.0 +bleach==6.2.0 +prompt-toolkit==3.0.43 +prompt_toolkit==3.0.50 +scipy==1.15.2 +gitdb==4.0.12 +notebook_shim==0.2.4 +zarr==2.18.3 +pyparsing==3.2.3 +pydantic==2.11.3 +tqdm==4.67.1 +nvidia-cufft-cu12==11.2.1.3 +sniffio==1.3.0 +diffusers==0.33.1 +typing-inspection==0.4.0 +jupyter_client==8.6.3 +gym-aloha==0.1.1 +Jinja2==3.1.6 +exceptiongroup==1.2.0 +stack-data==0.2.0 +TorchCodec==0.2.1 +numba==0.61.2 +smmap==5.0.2 +opencv-python==4.11.0.86 +debugpy==1.8.11 +tifffile==2025.3.30 +setproctitle==1.3.5 +imageio-ffmpeg==0.6.0 +overrides==7.4.0 +fasteners==0.19 +fsspec==2024.12.0 +jedi==0.19.2 +regex==2024.11.6 +jupyterlab_pygments==0.3.0 +jupyter_server==2.15.0 +anyio==4.6.2 +scikit-image==0.25.2 +pluggy==1.5.0 +xxhash==3.5.0 +lxml==5.3.2 +async-lru==2.0.4 +PyYAML==6.0.2 +tomli==2.0.1 +platformdirs==4.3.7 +nvidia-cuda-nvrtc-cu12==12.4.127 +psutil==7.0.0 +psutil==5.9.0 +mypy-extensions==1.0.0 +idna==3.7 +idna==3.10 +ipykernel==6.29.5 +nbformat==5.10.4 +charset-normalizer==3.3.2 +charset-normalizer==3.4.1 +pynput==1.8.1 +jupyterlab_server==2.27.3 +nvidia-nvtx-cu12==12.4.127 +sentry-sdk==2.25.1 +pyserial==3.5 +nvidia-curand-cu12==10.3.5.147 +attrs==25.3.0 +attrs==24.3.0 +GitPython==3.1.44 +rfc3986-validator==0.1.1 +jsonlines==4.0.0 +omegaconf==2.3.0 +typing_extensions==4.13.2 +typing_extensions==4.12.2 +cloudpickle==3.1.1 +tinycss2==1.4.0 +av==14.3.0 +jupyter-events==0.12.0 +imageio==2.37.0 +torchvision==0.21.0 +Brotli==1.0.9 +nvidia-cudnn-cu12==9.1.0.70 +traitlets==5.14.3 +huggingface-hub==0.30.2 +pydantic_core==2.33.1 +pyarrow==19.0.1 +jupyter_core==5.7.2 +deepdiff==8.4.2 +python-json-logger==3.2.1 +orderly-set==5.4.0 +pip==25.0 +jsonschema-specifications==2023.7.1 +ipython==8.30.0 +Send2Trash==1.8.2 +wheel==0.45.1 +prometheus_client==0.21.1 +absl-py==2.2.2 +mpmath==1.3.0 +torch==2.6.0 +dill==0.3.8 +termcolor==3.0.1 +gymnasium==0.29.1 +wrapt==1.17.2 +pandocfilters==1.5.0 +pillow==11.2.1 +h11==0.14.0 +gym-pusht==0.1.5 +nvidia-nvjitlink-cu12==12.4.127 +pymunk==6.11.1 +defusedxml==0.7.1 +nbclient==0.10.2 +fastjsonschema==2.20.0 +jupyter_server_terminals==0.4.4 +pyzmq==26.4.0 +nvidia-cublas-cu12==12.4.5.8 +networkx==3.4.2 +numpy==2.2.4 +toml==0.10.2 +filelock==3.18.0 +annotated-types==0.7.0 +PySocks==1.7.1 +draccus==0.10.0 +tornado==6.4.2 +mistune==3.1.2 +httpcore==1.0.2 +yarl==1.19.0 +mujoco==2.3.7 +aiohttp==3.11.16 +lazy_loader==0.4 +itsdangerous==2.2.0 +shapely==2.1.0 +protobuf==5.29.4 +httpx==0.27.0 +dm-control==1.0.14 +rpds-py==0.22.3 +aiohappyeyeballs==2.6.1 +Werkzeug==3.1.3 +typing-inspect==0.9.0 +decorator==5.1.1 +webencodings==0.5.1 +nvidia-nccl-cu12==2.21.5 +asttokens==3.0.0 +certifi==2025.1.31 +sympy==1.13.1 +dm-tree==0.1.9 +einops==0.8.1 +pexpect==4.8.0 +python-xlib==0.33 +pfzy==0.3.4 +inquirerpy==0.3.4 +MarkupSafe==3.0.2 +packaging==24.2 +transformers==4.51.2 +websocket-client==1.8.0 +docker-pycreds==0.4.0 +async-timeout==5.0.1 +rfc3339-validator==0.1.4 +jupyter-lsp==2.2.0 +six==1.17.0 +llvmlite==0.44.0 +pytest==8.3.5 +evdev==1.9.1 +feetech-servo-sdk==1.0.0 +glfw==2.8.0 +backports.tarfile==1.2.0 +inflect==7.3.1 +zipp==3.19.2 +jaraco.text==3.12.1 +autocommand==2.2.2 +typeguard==4.3.0 +wheel==0.43.0 +jaraco.collections==5.1.0 +tomli==2.0.1 +jaraco.context==5.3.0 +platformdirs==4.2.2 +more-itertools==10.3.0 +importlib_metadata==8.0.0 +jaraco.functools==4.0.1 +typing_extensions==4.12.2 +packaging==24.2 diff --git a/wandb/run-20250412_213619-shity_version/files/wandb-metadata.json b/wandb/run-20250412_213619-shity_version/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76fe73663d0f4bbe84f549489bf513e210b8439a --- /dev/null +++ b/wandb/run-20250412_213619-shity_version/files/wandb-metadata.json @@ -0,0 +1,52 @@ +{ + "os": "Linux-5.15.0-122-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.13", + "startedAt": "2025-04-12T21:36:19.372765Z", + "args": [ + "--policy.path=lerobot/pi0", + "--dataset.repo_id=maelic/hackathon7", + "--wandb.enable=true", + "--wandb.project=pi0_mistral_hackathon", + "--wandb.run_id=shity_version", + "--eval_freq=100", + "--save_freq=100", + "--output_dir=/scratch/train/pi_shity_version", + "--batch_size=20" + ], + "program": "/root/lerobot/lerobot/scripts/train.py", + "codePath": "lerobot/scripts/train.py", + "git": { + "remote": "https://github.com/huggingface/lerobot.git", + "commit": "974eef1b4fb8950e937d66fec275385976792269" + }, + "root": "/scratch/train/pi_shity_version", + "host": "scw-wizardly-cerf", + "executable": "/root/miniconda3/envs/lerobot/bin/python", + "codePathLocal": "lerobot/scripts/train.py", + "cpu_count": 24, + "cpu_count_logical": 24, + "gpu": "NVIDIA H100 PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "120949182464", + "used": "65785098240" + } + }, + "memory": { + "total": "253339410432" + }, + "cpu": { + "count": 24, + "countLogical": 24 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 PCIe", + "memoryTotal": "85520809984", + "cudaCores": 14592, + "architecture": "Hopper" + } + ], + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/wandb/run-20250412_213619-shity_version/files/wandb-summary.json b/wandb/run-20250412_213619-shity_version/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..08ff2da22cf39a13349a09c0786ee841629a29ae --- /dev/null +++ b/wandb/run-20250412_213619-shity_version/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":739},"train/loss":0.02187148805242032,"train/lr":1.252497502497502e-05,"_timestamp":1.7444945033926659e+09,"_step":600,"train/episodes":31.824611032531823,"train/grad_norm":0.6337687414884567,"train/samples":12000,"train/dataloading_s":0.005114598170102909,"train/update_s":0.7455780633199993,"train/steps":600,"_runtime":739.988406225,"train/epochs":1.0608203677510608,"train/l2_loss":0.017834482714533806} \ No newline at end of file diff --git a/wandb/run-20250412_213619-shity_version/logs/debug-core.log b/wandb/run-20250412_213619-shity_version/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..579ad6bd42ffad0c42ca96dfb76f205944cb1c9b --- /dev/null +++ b/wandb/run-20250412_213619-shity_version/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2025-04-12T21:36:19.177403034Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpwk0ga7_d/port-26690.txt","pid":26690,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-12T21:36:19.180223177Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":26690} +{"time":"2025-04-12T21:36:19.180219091Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43653,"Zone":""}} +{"time":"2025-04-12T21:36:19.368454512Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:57530"} +{"time":"2025-04-12T21:36:19.374011754Z","level":"INFO","msg":"handleInformInit: received","streamId":"shity_version","id":"127.0.0.1:57530"} +{"time":"2025-04-12T21:36:19.636403544Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"shity_version","id":"127.0.0.1:57530"} +{"time":"2025-04-12T21:48:39.361071747Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:57530"} +{"time":"2025-04-12T21:48:39.361145107Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:57530"} +{"time":"2025-04-12T21:48:39.361182012Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-04-12T21:48:39.361253588Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:57530"} +{"time":"2025-04-12T21:48:40.060781775Z","level":"ERROR","msg":"processOutgoingData: flush error","error":"write tcp 127.0.0.1:43653->127.0.0.1:57530: use of closed network connection","id":"127.0.0.1:57530"} +{"time":"2025-04-12T21:48:40.927435363Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/wandb/run-20250412_213619-shity_version/logs/debug-internal.log b/wandb/run-20250412_213619-shity_version/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e2feb0060528f4575a8b4897c821339947d67fa4 --- /dev/null +++ b/wandb/run-20250412_213619-shity_version/logs/debug-internal.log @@ -0,0 +1,10 @@ +{"time":"2025-04-12T21:36:19.374153667Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/scratch/train/pi_shity_version/wandb/run-20250412_213619-shity_version/logs/debug-core.log"} +{"time":"2025-04-12T21:36:19.636350885Z","level":"INFO","msg":"created new stream","id":"shity_version"} +{"time":"2025-04-12T21:36:19.636398405Z","level":"INFO","msg":"stream: started","id":"shity_version"} +{"time":"2025-04-12T21:36:19.636436233Z","level":"INFO","msg":"handler: started","stream_id":"shity_version"} +{"time":"2025-04-12T21:36:19.636435241Z","level":"INFO","msg":"writer: Do: started","stream_id":"shity_version"} +{"time":"2025-04-12T21:36:19.636464184Z","level":"INFO","msg":"sender: started","stream_id":"shity_version"} +{"time":"2025-04-12T21:36:19.770119947Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-12T21:48:39.361156513Z","level":"INFO","msg":"stream: closing","id":"shity_version"} +{"time":"2025-04-12T21:48:39.36118102Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-12T21:48:39.361222803Z","level":"INFO","msg":"Stopped system monitor"} diff --git a/wandb/run-20250412_213619-shity_version/logs/debug.log b/wandb/run-20250412_213619-shity_version/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..01535eb8e847bce603e696947a9e132b01c31102 --- /dev/null +++ b/wandb/run-20250412_213619-shity_version/logs/debug.log @@ -0,0 +1,887 @@ +2025-04-12 21:36:19,369 INFO MainThread:26690 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-04-12 21:36:19,370 INFO MainThread:26690 [wandb_setup.py:_flush():67] Configure stats pid to 26690 +2025-04-12 21:36:19,370 INFO MainThread:26690 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-04-12 21:36:19,370 INFO MainThread:26690 [wandb_setup.py:_flush():67] Loading settings from /root/lerobot/wandb/settings +2025-04-12 21:36:19,370 INFO MainThread:26690 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-12 21:36:19,370 INFO MainThread:26690 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /scratch/train/pi_shity_version/wandb/run-20250412_213619-shity_version/logs/debug.log +2025-04-12 21:36:19,370 INFO MainThread:26690 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /scratch/train/pi_shity_version/wandb/run-20250412_213619-shity_version/logs/debug-internal.log +2025-04-12 21:36:19,370 INFO MainThread:26690 [wandb_init.py:init():781] calling init triggers +2025-04-12 21:36:19,370 INFO MainThread:26690 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'maelic/hackathon7', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'pi0', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 50, 'n_action_steps': 50, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [224, 224], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'proj_width': 1024, 'num_steps': 10, 'use_cache': True, 'attention_implementation': 'eager', 'freeze_vision_encoder': True, 'train_expert_only': False, 'train_state_proj': True, 'optimizer_lr': 2.5e-05, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-10, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06}, 'output_dir': '/scratch/train/pi_shity_version', 'job_name': 'pi0', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 20, 'steps': 100000, 'eval_freq': 100, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 100, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 2.5e-05, 'weight_decay': 1e-10, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 2.5e-05, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'pi0_mistral_hackathon', 'entity': None, 'notes': None, 'run_id': 'shity_version', 'mode': None}, '_wandb': {}} +2025-04-12 21:36:19,370 INFO MainThread:26690 [wandb_init.py:init():809] starting backend +2025-04-12 21:36:19,370 INFO MainThread:26690 [wandb_init.py:init():813] sending inform_init request +2025-04-12 21:36:19,372 INFO MainThread:26690 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-12 21:36:19,372 INFO MainThread:26690 [wandb_init.py:init():823] backend started and connected +2025-04-12 21:36:19,373 INFO MainThread:26690 [wandb_init.py:init():915] updated telemetry +2025-04-12 21:36:19,377 INFO MainThread:26690 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-04-12 21:36:19,768 INFO MainThread:26690 [wandb_init.py:init():1014] starting run threads in backend +2025-04-12 21:36:19,867 INFO MainThread:26690 [wandb_run.py:_console_start():2454] atexit reg +2025-04-12 21:36:19,867 INFO MainThread:26690 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-04-12 21:36:19,867 INFO MainThread:26690 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-04-12 21:36:19,867 INFO MainThread:26690 [wandb_run.py:_redirect():2394] Redirects installed. +2025-04-12 21:36:19,869 INFO MainThread:26690 [wandb_init.py:init():1056] run started, returning control to user process +2025-04-12 21:48:39,360 INFO MsgRouterThr:26690 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 3 handles. +2025-04-12 21:48:39,933 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,935 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,935 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,935 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,935 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,935 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,936 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,936 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,936 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,936 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,936 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,936 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,937 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,937 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,937 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,937 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,937 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,937 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,938 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,938 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,938 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,938 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,938 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,938 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,939 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,939 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,939 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,939 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,939 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,939 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,940 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-04-12 21:48:39,940 ERROR MainThread:26690 [redirect.py:_on_write():661] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2381, in + lambda data: self._console_raw_callback("stderr", data), + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe diff --git a/wandb/run-20250412_213619-shity_version/run-shity_version.wandb b/wandb/run-20250412_213619-shity_version/run-shity_version.wandb new file mode 100644 index 0000000000000000000000000000000000000000..99dda00af40c3af4fed8abffd52c7ee360aba97a Binary files /dev/null and b/wandb/run-20250412_213619-shity_version/run-shity_version.wandb differ diff --git a/wandb/run-20250412_214845-shity_version/files/config.yaml b/wandb/run-20250412_214845-shity_version/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f09c72fee93558daac54fa70dc5e58b2bd50b901 --- /dev/null +++ b/wandb/run-20250412_214845-shity_version/files/config.yaml @@ -0,0 +1,163 @@ +_wandb: + value: + cli_version: 0.19.9 + m: [] + python_version: 3.10.13 + t: + "1": + - 1 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "3": + - 5 + - 13 + - 14 + - 15 + - 16 + - 23 + - 55 + - 62 + "4": 3.10.13 + "5": 0.19.9 + "8": + - 5 + "12": 0.19.9 + "13": linux-x86_64 +batch_size: + value: 20 +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: maelic/hackathon7 + revision: null + root: null + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 100 +job_name: + value: pi0 +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: null +output_dir: + value: /scratch/train/pi_shity_version +policy: + value: + adapt_to_pi_aloha: false + attention_implementation: eager + chunk_size: 50 + device: cuda + empty_cameras: 0 + freeze_vision_encoder: true + max_action_dim: 32 + max_state_dim: 32 + n_action_steps: 50 + n_obs_steps: 1 + normalization_mapping: + ACTION: MEAN_STD + STATE: MEAN_STD + VISUAL: IDENTITY + num_steps: 10 + optimizer_betas: + - 0.9 + - 0.95 + optimizer_eps: 1e-08 + optimizer_lr: 2.5e-05 + optimizer_weight_decay: 1e-10 + proj_width: 1024 + resize_imgs_with_padding: + - 224 + - 224 + scheduler_decay_lr: 2.5e-06 + scheduler_decay_steps: 30000 + scheduler_warmup_steps: 1000 + tokenizer_max_length: 48 + train_expert_only: false + train_state_proj: true + type: pi0 + use_amp: false + use_cache: true + use_delta_joint_actions_aloha: false +resume: + value: true +save_checkpoint: + value: true +save_freq: + value: 100 +scheduler: + value: null +seed: + value: 1000 +steps: + value: 100000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: true + enable: true + entity: null + mode: null + notes: null + project: pi0_mistral_hackathon + run_id: shity_version diff --git a/wandb/run-20250412_214845-shity_version/files/output.log b/wandb/run-20250412_214845-shity_version/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..cdbdc7fabd01cb2d3d4cbadb978c7b2200db2be1 --- /dev/null +++ b/wandb/run-20250412_214845-shity_version/files/output.log @@ -0,0 +1,26 @@ +Logs will be synced with wandb. +INFO 2025-04-12 21:48:46 ndb_utils.py:96 Track this run --> https://wandb.ai/helper2424-3commas/pi0_mistral_hackathon/runs/shity_version +INFO 2025-04-12 21:48:46 ts/train.py:127 Creating dataset +Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 30/30 [00:00<00:00, 241979.08it/s] +INFO 2025-04-12 21:48:46 ts/train.py:138 Creating policy +INFO 2025-04-12 21:49:34 ts/train.py:144 Creating optimizer and scheduler +Traceback (most recent call last): + File "/root/lerobot/lerobot/scripts/train.py", line 288, in + train() + File "/root/lerobot/lerobot/configs/parser.py", line 227, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/root/lerobot/lerobot/scripts/train.py", line 145, in train + optimizer, lr_scheduler = make_optimizer_and_scheduler(cfg, policy) + File "/root/lerobot/lerobot/common/optim/factory.py", line 38, in make_optimizer_and_scheduler + optimizer = cfg.optimizer.build(params) +AttributeError: 'NoneType' object has no attribute 'build' +Traceback (most recent call last): + File "/root/lerobot/lerobot/scripts/train.py", line 288, in + train() + File "/root/lerobot/lerobot/configs/parser.py", line 227, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/root/lerobot/lerobot/scripts/train.py", line 145, in train + optimizer, lr_scheduler = make_optimizer_and_scheduler(cfg, policy) + File "/root/lerobot/lerobot/common/optim/factory.py", line 38, in make_optimizer_and_scheduler + optimizer = cfg.optimizer.build(params) +AttributeError: 'NoneType' object has no attribute 'build' diff --git a/wandb/run-20250412_214845-shity_version/files/requirements.txt b/wandb/run-20250412_214845-shity_version/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..19233a5893fbe938d47f546a80a53474123ed3f2 --- /dev/null +++ b/wandb/run-20250412_214845-shity_version/files/requirements.txt @@ -0,0 +1,245 @@ +tokenizers==0.21.1 +frozenlist==1.5.0 +numcodecs==0.13.1 +terminado==0.17.1 +Farama-Notifications==0.0.4 +labmaze==1.0.6 +Flask==3.1.0 +nvidia-cusparse-cu12==12.3.1.170 +jsonschema==4.23.0 +pycparser==2.21 +pycparser==2.22 +argon2-cffi==21.3.0 +multidict==6.4.3 +safetensors==0.5.3 +setuptools==75.8.0 +multiprocess==0.70.16 +urllib3==2.4.0 +urllib3==2.3.0 +gdown==5.2.0 +pytz==2025.2 +beautifulsoup4==4.12.3 +beautifulsoup4==4.13.3 +importlib_metadata==8.6.1 +babel==2.16.0 +cmake==4.0.0 +pyyaml-include==1.4.1 +asciitree==0.3.3 +h5py==3.13.0 +antlr4-python3-runtime==4.9.3 +tzdata==2025.2 +jupyterlab==4.3.4 +datasets==3.5.0 +mergedeep==1.3.4 +nvidia-cusparselt-cu12==0.6.2 +comm==0.2.1 +blinker==1.9.0 +referencing==0.30.2 +nvidia-cuda-cupti-cu12==12.4.127 +nbconvert==7.16.6 +iniconfig==2.1.0 +click==8.1.8 +nest-asyncio==1.6.0 +rerun-sdk==0.22.1 +wcwidth==0.2.5 +wcwidth==0.2.13 +wandb==0.19.9 +argon2-cffi-bindings==21.2.0 +nvidia-cusolver-cu12==11.6.1.9 +ptyprocess==0.7.0 +triton==3.2.0 +parso==0.8.4 +cffi==1.17.1 +executing==0.8.3 +soupsieve==2.6 +soupsieve==2.5 +lerobot==0.1.0 +nvidia-cuda-runtime-cu12==12.4.127 +python-dateutil==2.9.0.post0 +pure-eval==0.2.2 +aiosignal==1.3.2 +hf_transfer==0.1.9 +pandas==2.2.3 +dm-env==1.6 +pygame==2.6.1 +opencv-python-headless==4.11.0.86 +PyOpenGL==3.1.9 +matplotlib-inline==0.1.6 +requests==2.32.3 +propcache==0.3.1 +json5==0.9.25 +Pygments==2.15.1 +zipp==3.21.0 +bleach==6.2.0 +prompt-toolkit==3.0.43 +prompt_toolkit==3.0.50 +scipy==1.15.2 +gitdb==4.0.12 +notebook_shim==0.2.4 +zarr==2.18.3 +pyparsing==3.2.3 +pydantic==2.11.3 +tqdm==4.67.1 +nvidia-cufft-cu12==11.2.1.3 +sniffio==1.3.0 +diffusers==0.33.1 +typing-inspection==0.4.0 +jupyter_client==8.6.3 +gym-aloha==0.1.1 +Jinja2==3.1.6 +exceptiongroup==1.2.0 +stack-data==0.2.0 +TorchCodec==0.2.1 +numba==0.61.2 +smmap==5.0.2 +opencv-python==4.11.0.86 +debugpy==1.8.11 +tifffile==2025.3.30 +setproctitle==1.3.5 +imageio-ffmpeg==0.6.0 +overrides==7.4.0 +fasteners==0.19 +fsspec==2024.12.0 +jedi==0.19.2 +regex==2024.11.6 +jupyterlab_pygments==0.3.0 +jupyter_server==2.15.0 +anyio==4.6.2 +scikit-image==0.25.2 +pluggy==1.5.0 +xxhash==3.5.0 +lxml==5.3.2 +async-lru==2.0.4 +PyYAML==6.0.2 +tomli==2.0.1 +platformdirs==4.3.7 +nvidia-cuda-nvrtc-cu12==12.4.127 +psutil==7.0.0 +psutil==5.9.0 +mypy-extensions==1.0.0 +idna==3.7 +idna==3.10 +ipykernel==6.29.5 +nbformat==5.10.4 +charset-normalizer==3.3.2 +charset-normalizer==3.4.1 +pynput==1.8.1 +jupyterlab_server==2.27.3 +nvidia-nvtx-cu12==12.4.127 +sentry-sdk==2.25.1 +pyserial==3.5 +nvidia-curand-cu12==10.3.5.147 +attrs==25.3.0 +attrs==24.3.0 +GitPython==3.1.44 +rfc3986-validator==0.1.1 +jsonlines==4.0.0 +omegaconf==2.3.0 +typing_extensions==4.13.2 +typing_extensions==4.12.2 +cloudpickle==3.1.1 +tinycss2==1.4.0 +av==14.3.0 +jupyter-events==0.12.0 +imageio==2.37.0 +torchvision==0.21.0 +Brotli==1.0.9 +nvidia-cudnn-cu12==9.1.0.70 +traitlets==5.14.3 +huggingface-hub==0.30.2 +pydantic_core==2.33.1 +pyarrow==19.0.1 +jupyter_core==5.7.2 +deepdiff==8.4.2 +python-json-logger==3.2.1 +orderly-set==5.4.0 +pip==25.0 +jsonschema-specifications==2023.7.1 +ipython==8.30.0 +Send2Trash==1.8.2 +wheel==0.45.1 +prometheus_client==0.21.1 +absl-py==2.2.2 +mpmath==1.3.0 +torch==2.6.0 +dill==0.3.8 +termcolor==3.0.1 +gymnasium==0.29.1 +wrapt==1.17.2 +pandocfilters==1.5.0 +pillow==11.2.1 +h11==0.14.0 +gym-pusht==0.1.5 +nvidia-nvjitlink-cu12==12.4.127 +pymunk==6.11.1 +defusedxml==0.7.1 +nbclient==0.10.2 +fastjsonschema==2.20.0 +jupyter_server_terminals==0.4.4 +pyzmq==26.4.0 +nvidia-cublas-cu12==12.4.5.8 +networkx==3.4.2 +numpy==2.2.4 +toml==0.10.2 +filelock==3.18.0 +annotated-types==0.7.0 +PySocks==1.7.1 +draccus==0.10.0 +tornado==6.4.2 +mistune==3.1.2 +httpcore==1.0.2 +yarl==1.19.0 +mujoco==2.3.7 +aiohttp==3.11.16 +lazy_loader==0.4 +itsdangerous==2.2.0 +shapely==2.1.0 +protobuf==5.29.4 +httpx==0.27.0 +dm-control==1.0.14 +rpds-py==0.22.3 +aiohappyeyeballs==2.6.1 +Werkzeug==3.1.3 +typing-inspect==0.9.0 +decorator==5.1.1 +webencodings==0.5.1 +nvidia-nccl-cu12==2.21.5 +asttokens==3.0.0 +certifi==2025.1.31 +sympy==1.13.1 +dm-tree==0.1.9 +einops==0.8.1 +pexpect==4.8.0 +python-xlib==0.33 +pfzy==0.3.4 +inquirerpy==0.3.4 +MarkupSafe==3.0.2 +packaging==24.2 +transformers==4.51.2 +websocket-client==1.8.0 +docker-pycreds==0.4.0 +async-timeout==5.0.1 +rfc3339-validator==0.1.4 +jupyter-lsp==2.2.0 +six==1.17.0 +llvmlite==0.44.0 +pytest==8.3.5 +evdev==1.9.1 +feetech-servo-sdk==1.0.0 +glfw==2.8.0 +backports.tarfile==1.2.0 +inflect==7.3.1 +zipp==3.19.2 +jaraco.text==3.12.1 +autocommand==2.2.2 +typeguard==4.3.0 +wheel==0.43.0 +jaraco.collections==5.1.0 +tomli==2.0.1 +jaraco.context==5.3.0 +platformdirs==4.2.2 +more-itertools==10.3.0 +importlib_metadata==8.0.0 +jaraco.functools==4.0.1 +typing_extensions==4.12.2 +packaging==24.2 diff --git a/wandb/run-20250412_214845-shity_version/files/wandb-metadata.json b/wandb/run-20250412_214845-shity_version/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d55de3a5bfc416a8687ba3a4f69040c0a7dacb39 --- /dev/null +++ b/wandb/run-20250412_214845-shity_version/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-122-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.13", + "startedAt": "2025-04-12T21:48:45.537686Z", + "args": [ + "--policy.path=lerobot/pi0", + "--dataset.repo_id=maelic/hackathon7", + "--wandb.enable=true", + "--wandb.project=pi0_mistral_hackathon", + "--wandb.run_id=shity_version", + "--eval_freq=100", + "--save_freq=100", + "--output_dir=/scratch/train/pi_shity_version", + "--batch_size=20", + "--wandb.disable_artifact=true", + "--resume=true" + ], + "program": "/root/lerobot/lerobot/scripts/train.py", + "codePath": "lerobot/scripts/train.py", + "git": { + "remote": "https://github.com/huggingface/lerobot.git", + "commit": "974eef1b4fb8950e937d66fec275385976792269" + }, + "root": "/scratch/train/pi_shity_version", + "host": "scw-wizardly-cerf", + "executable": "/root/miniconda3/envs/lerobot/bin/python", + "codePathLocal": "lerobot/scripts/train.py", + "cpu_count": 24, + "cpu_count_logical": 24, + "gpu": "NVIDIA H100 PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "120949182464", + "used": "111001305088" + } + }, + "memory": { + "total": "253339410432" + }, + "cpu": { + "count": 24, + "countLogical": 24 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 PCIe", + "memoryTotal": "85520809984", + "cudaCores": 14592, + "architecture": "Hopper" + } + ], + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/wandb/run-20250412_214845-shity_version/files/wandb-summary.json b/wandb/run-20250412_214845-shity_version/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..030333742139db505af09858355e09ff08e2b471 --- /dev/null +++ b/wandb/run-20250412_214845-shity_version/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/grad_norm":1.3849487748742104,"train/update_s":0.7492876601800253,"_timestamp":1.74449402504731e+09,"train/episodes":10.608203677510607,"train/epochs":0.3536067892503536,"train/loss":0.06476064555346966,"train/samples":4000,"_wandb":{"runtime":48},"train/steps":200,"train/lr":2.534965034965035e-06,"train/l2_loss":0.05327819660305977,"_step":200,"train/dataloading_s":0.007002105624987962,"_runtime":486.103999596} \ No newline at end of file diff --git a/wandb/run-20250412_214845-shity_version/logs/debug-core.log b/wandb/run-20250412_214845-shity_version/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..dc03ebd5a06e1db91bf57baa79672bba4ba0ff10 --- /dev/null +++ b/wandb/run-20250412_214845-shity_version/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-04-12T21:48:45.341638779Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpcitlo9oe/port-26962.txt","pid":26962,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-12T21:48:45.344118424Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":26962} +{"time":"2025-04-12T21:48:45.344090181Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44835,"Zone":""}} +{"time":"2025-04-12T21:48:45.533081667Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:52586"} +{"time":"2025-04-12T21:48:45.538910504Z","level":"INFO","msg":"handleInformInit: received","streamId":"shity_version","id":"127.0.0.1:52586"} +{"time":"2025-04-12T21:48:45.768056272Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"shity_version","id":"127.0.0.1:52586"} +{"time":"2025-04-12T21:49:34.499014913Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:52586"} +{"time":"2025-04-12T21:49:34.499093881Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:52586"} +{"time":"2025-04-12T21:49:34.499128172Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-04-12T21:49:34.499222463Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:52586"} +{"time":"2025-04-12T21:49:35.230519198Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:52586"} +{"time":"2025-04-12T21:49:35.230535702Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:52586"} +{"time":"2025-04-12T21:49:35.230545217Z","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20250412_214845-shity_version/logs/debug-internal.log b/wandb/run-20250412_214845-shity_version/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9fd275b92beb77a0fe030abb9b335815b6c3042d --- /dev/null +++ b/wandb/run-20250412_214845-shity_version/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-04-12T21:48:45.539036331Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/scratch/train/pi_shity_version/wandb/run-20250412_214845-shity_version/logs/debug-core.log"} +{"time":"2025-04-12T21:48:45.767939487Z","level":"INFO","msg":"created new stream","id":"shity_version"} +{"time":"2025-04-12T21:48:45.768051014Z","level":"INFO","msg":"stream: started","id":"shity_version"} +{"time":"2025-04-12T21:48:45.768079557Z","level":"INFO","msg":"writer: Do: started","stream_id":"shity_version"} +{"time":"2025-04-12T21:48:45.768102271Z","level":"INFO","msg":"sender: started","stream_id":"shity_version"} +{"time":"2025-04-12T21:48:45.768084685Z","level":"INFO","msg":"handler: started","stream_id":"shity_version"} +{"time":"2025-04-12T21:48:46.179700168Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-12T21:49:34.499107401Z","level":"INFO","msg":"stream: closing","id":"shity_version"} +{"time":"2025-04-12T21:49:34.49913363Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-12T21:49:34.499169854Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-12T21:49:35.088264035Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-12T21:49:35.230336885Z","level":"INFO","msg":"handler: closed","stream_id":"shity_version"} +{"time":"2025-04-12T21:49:35.2303801Z","level":"INFO","msg":"sender: closed","stream_id":"shity_version"} +{"time":"2025-04-12T21:49:35.23037318Z","level":"INFO","msg":"writer: Close: closed","stream_id":"shity_version"} +{"time":"2025-04-12T21:49:35.230452087Z","level":"INFO","msg":"stream: closed","id":"shity_version"} diff --git a/wandb/run-20250412_214845-shity_version/logs/debug.log b/wandb/run-20250412_214845-shity_version/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3336bcf4569aa37ecd4a4785fae4d8a57c2dbc99 --- /dev/null +++ b/wandb/run-20250412_214845-shity_version/logs/debug.log @@ -0,0 +1,24 @@ +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_setup.py:_flush():67] Configure stats pid to 26962 +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_setup.py:_flush():67] Loading settings from /root/lerobot/wandb/settings +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /scratch/train/pi_shity_version/wandb/run-20250412_214845-shity_version/logs/debug.log +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /scratch/train/pi_shity_version/wandb/run-20250412_214845-shity_version/logs/debug-internal.log +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_init.py:init():781] calling init triggers +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'maelic/hackathon7', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'pi0', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 50, 'n_action_steps': 50, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [224, 224], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'proj_width': 1024, 'num_steps': 10, 'use_cache': True, 'attention_implementation': 'eager', 'freeze_vision_encoder': True, 'train_expert_only': False, 'train_state_proj': True, 'optimizer_lr': 2.5e-05, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-10, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06}, 'output_dir': '/scratch/train/pi_shity_version', 'job_name': 'pi0', 'resume': True, 'seed': 1000, 'num_workers': 4, 'batch_size': 20, 'steps': 100000, 'eval_freq': 100, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 100, 'use_policy_training_preset': True, 'optimizer': None, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'pi0_mistral_hackathon', 'entity': None, 'notes': None, 'run_id': 'shity_version', 'mode': None}, '_wandb': {}} +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_init.py:init():809] starting backend +2025-04-12 21:48:45,534 INFO MainThread:26962 [wandb_init.py:init():813] sending inform_init request +2025-04-12 21:48:45,537 INFO MainThread:26962 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-12 21:48:45,537 INFO MainThread:26962 [wandb_init.py:init():823] backend started and connected +2025-04-12 21:48:45,538 INFO MainThread:26962 [wandb_init.py:init():915] updated telemetry +2025-04-12 21:48:45,542 INFO MainThread:26962 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-04-12 21:48:46,176 INFO MainThread:26962 [wandb_init.py:init():1009] run resumed +2025-04-12 21:48:46,178 INFO MainThread:26962 [wandb_init.py:init():1014] starting run threads in backend +2025-04-12 21:48:46,276 INFO MainThread:26962 [wandb_run.py:_console_start():2454] atexit reg +2025-04-12 21:48:46,276 INFO MainThread:26962 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-04-12 21:48:46,276 INFO MainThread:26962 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-04-12 21:48:46,276 INFO MainThread:26962 [wandb_run.py:_redirect():2394] Redirects installed. +2025-04-12 21:48:46,278 INFO MainThread:26962 [wandb_init.py:init():1056] run started, returning control to user process +2025-04-12 21:49:34,498 INFO MsgRouterThr:26962 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20250412_214845-shity_version/run-shity_version.wandb b/wandb/run-20250412_214845-shity_version/run-shity_version.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7d1cfbe3b4e0b9b57dd64f636816c384f306aaf6 Binary files /dev/null and b/wandb/run-20250412_214845-shity_version/run-shity_version.wandb differ