diff --git "a/debugger_ovh_transformers.ipynb" "b/debugger_ovh_transformers.ipynb" --- "a/debugger_ovh_transformers.ipynb" +++ "b/debugger_ovh_transformers.ipynb" @@ -16668,7 +16668,7535 @@ " 44%|█████████████████▌ | 820/1864 [06:33<07:47, 2.24it/s]\u001b[A\n", " 44%|█████████████████▌ | 821/1864 [06:33<07:47, 2.23it/s]\u001b[A\n", " 44%|█████████████████▋ | 822/1864 [06:34<08:28, 2.05it/s]\u001b[A\n", - " 44%|█████████████████▋ | 823/1864 [06:34<08:08, 2.13it/s]\u001b[A" + " 44%|█████████████████▋ | 823/1864 [06:34<08:08, 2.13it/s]\u001b[A\n", + " 44%|█████████████████▋ | 824/1864 [06:34<07:46, 2.23it/s]\u001b[A\n", + " 44%|█████████████████▋ | 825/1864 [06:35<08:49, 1.96it/s]\u001b[A\n", + " 44%|█████████████████▋ | 826/1864 [06:36<09:27, 1.83it/s]\u001b[A\n", + " 44%|█████████████████▋ | 827/1864 [06:36<09:45, 1.77it/s]\u001b[A\n", + " 44%|█████████████████▊ | 828/1864 [06:37<09:04, 1.90it/s]\u001b[A\n", + " 44%|█████████████████▊ | 829/1864 [06:37<08:50, 1.95it/s]\u001b[A\n", + " 45%|█████████████████▊ | 830/1864 [06:38<09:40, 1.78it/s]\u001b[A\n", + " 45%|█████████████████▊ | 831/1864 [06:38<09:39, 1.78it/s]\u001b[A\n", + " 45%|█████████████████▊ | 832/1864 [06:39<09:32, 1.80it/s]\u001b[A\n", + " 45%|█████████████████▉ | 833/1864 [06:39<09:07, 1.88it/s]\u001b[A\n", + " 45%|█████████████████▉ | 834/1864 [06:40<09:08, 1.88it/s]\u001b[A\n", + " 45%|█████████████████▉ | 835/1864 [06:40<09:00, 1.91it/s]\u001b[A\n", + " 45%|█████████████████▉ | 836/1864 [06:41<08:30, 2.01it/s]\u001b[A\n", + " 45%|█████████████████▉ | 837/1864 [06:41<08:42, 1.96it/s]\u001b[A\n", + " 45%|█████████████████▉ | 838/1864 [06:42<08:23, 2.04it/s]\u001b[A\n", + " 45%|██████████████████ | 839/1864 [06:42<08:46, 1.95it/s]\u001b[A\n", + " 45%|██████████████████ | 840/1864 [06:43<08:31, 2.00it/s]\u001b[A\n", + " 45%|██████████████████ | 841/1864 [06:43<08:39, 1.97it/s]\u001b[A\n", + " 45%|██████████████████ | 842/1864 [06:44<08:39, 1.97it/s]\u001b[A\n", + " 45%|██████████████████ | 843/1864 [06:44<08:19, 2.04it/s]\u001b[A\n", + " 45%|██████████████████ | 844/1864 [06:45<08:14, 2.06it/s]\u001b[A\n", + " 45%|██████████████████▏ | 845/1864 [06:45<08:12, 2.07it/s]\u001b[A\n", + " 45%|██████████████████▏ | 846/1864 [06:46<08:24, 2.02it/s]\u001b[A\n", + " 45%|██████████████████▏ | 847/1864 [06:46<08:26, 2.01it/s]\u001b[A\n", + " 45%|██████████████████▏ | 848/1864 [06:47<08:32, 1.98it/s]\u001b[A\n", + " 46%|██████████████████▏ | 849/1864 [06:47<08:16, 2.04it/s]\u001b[A\n", + " 46%|██████████████████▏ | 850/1864 [06:48<08:41, 1.94it/s]\u001b[A\n", + " 46%|██████████████████▎ | 851/1864 [06:48<08:19, 2.03it/s]\u001b[A\n", + " 46%|██████████████████▎ | 852/1864 [06:49<08:13, 2.05it/s]\u001b[A\n", + " 46%|██████████████████▎ | 853/1864 [06:49<07:54, 2.13it/s]\u001b[A\n", + " 46%|██████████████████▎ | 854/1864 [06:50<08:11, 2.06it/s]\u001b[A\n", + " 46%|██████████████████▎ | 855/1864 [06:50<08:52, 1.90it/s]\u001b[A\n", + " 46%|██████████████████▎ | 856/1864 [06:51<08:58, 1.87it/s]\u001b[A\n", + " 46%|██████████████████▍ | 857/1864 [06:51<08:34, 1.96it/s]\u001b[A\n", + " 46%|██████████████████▍ | 858/1864 [06:52<08:02, 2.08it/s]\u001b[A\n", + " 46%|██████████████████▍ | 859/1864 [06:52<07:45, 2.16it/s]\u001b[A\n", + " 46%|██████████████████▍ | 860/1864 [06:53<08:06, 2.06it/s]\u001b[A\n", + " 46%|██████████████████▍ | 861/1864 [06:53<08:14, 2.03it/s]\u001b[A\n", + " 46%|██████████████████▍ | 862/1864 [06:54<08:24, 1.98it/s]\u001b[A\n", + " 46%|██████████████████▌ | 863/1864 [06:54<08:16, 2.02it/s]\u001b[A\n", + " 46%|██████████████████▌ | 864/1864 [06:55<08:12, 2.03it/s]\u001b[A\n", + " 46%|██████████████████▌ | 865/1864 [06:55<07:40, 2.17it/s]\u001b[A\n", + " 46%|██████████████████▌ | 866/1864 [06:56<07:37, 2.18it/s]\u001b[A\n", + " 47%|██████████████████▌ | 867/1864 [06:56<07:28, 2.22it/s]\u001b[A\n", + " 47%|██████████████████▋ | 868/1864 [06:57<07:58, 2.08it/s]\u001b[A\n", + " 47%|██████████████████▋ | 869/1864 [06:57<08:49, 1.88it/s]\u001b[A\n", + " 47%|██████████████████▋ | 870/1864 [06:58<08:14, 2.01it/s]\u001b[A\n", + " 47%|██████████████████▋ | 871/1864 [06:58<08:28, 1.95it/s]\u001b[A\n", + " 47%|██████████████████▋ | 872/1864 [06:59<08:20, 1.98it/s]\u001b[A\n", + " 47%|██████████████████▋ | 873/1864 [06:59<08:55, 1.85it/s]\u001b[A\n", + " 47%|██████████████████▊ | 874/1864 [07:00<08:32, 1.93it/s]\u001b[A\n", + " 47%|██████████████████▊ | 875/1864 [07:00<08:37, 1.91it/s]\u001b[A\n", + " 47%|██████████████████▊ | 876/1864 [07:01<08:20, 1.97it/s]\u001b[A\n", + " 47%|██████████████████▊ | 877/1864 [07:01<08:14, 1.99it/s]\u001b[A\n", + " 47%|██████████████████▊ | 878/1864 [07:02<08:27, 1.94it/s]\u001b[A\n", + " 47%|██████████████████▊ | 879/1864 [07:02<08:07, 2.02it/s]\u001b[A\n", + " 47%|██████████████████▉ | 880/1864 [07:03<07:47, 2.10it/s]\u001b[A\n", + " 47%|██████████████████▉ | 881/1864 [07:03<07:50, 2.09it/s]\u001b[A\n", + " 47%|██████████████████▉ | 882/1864 [07:04<08:09, 2.01it/s]\u001b[A\n", + " 47%|██████████████████▉ | 883/1864 [07:04<07:58, 2.05it/s]\u001b[A\n", + " 47%|██████████████████▉ | 884/1864 [07:05<07:47, 2.10it/s]\u001b[A\n", + " 47%|██████████████████▉ | 885/1864 [07:05<07:51, 2.07it/s]\u001b[A\n", + " 48%|███████████████████ | 886/1864 [07:06<07:44, 2.11it/s]\u001b[A\n", + " 48%|███████████████████ | 887/1864 [07:06<07:40, 2.12it/s]\u001b[A\n", + " 48%|███████████████████ | 888/1864 [07:07<08:13, 1.98it/s]\u001b[A\n", + " 48%|███████████████████ | 889/1864 [07:07<08:48, 1.85it/s]\u001b[A\n", + " 48%|███████████████████ | 890/1864 [07:08<08:54, 1.82it/s]\u001b[A\n", + " 48%|███████████████████ | 891/1864 [07:08<09:12, 1.76it/s]\u001b[A\n", + " 48%|███████████████████▏ | 892/1864 [07:09<09:14, 1.75it/s]\u001b[A\n", + " 48%|███████████████████▏ | 893/1864 [07:10<08:57, 1.81it/s]\u001b[A\n", + " 48%|███████████████████▏ | 894/1864 [07:10<08:32, 1.89it/s]\u001b[A\n", + " 48%|███████████████████▏ | 895/1864 [07:11<08:14, 1.96it/s]\u001b[A\n", + " 48%|███████████████████▏ | 896/1864 [07:11<07:56, 2.03it/s]\u001b[A\n", + " 48%|███████████████████▏ | 897/1864 [07:12<08:09, 1.98it/s]\u001b[A\n", + " 48%|███████████████████▎ | 898/1864 [07:12<07:50, 2.05it/s]\u001b[A\n", + " 48%|███████████████████▎ | 899/1864 [07:12<07:32, 2.13it/s]\u001b[A\n", + " 48%|███████████████████▎ | 900/1864 [07:13<07:45, 2.07it/s]\u001b[A\n", + " 48%|███████████████████▎ | 901/1864 [07:13<08:03, 1.99it/s]\u001b[A\n", + " 48%|███████████████████▎ | 902/1864 [07:14<08:02, 1.99it/s]\u001b[A\n", + " 48%|███████████████████▍ | 903/1864 [07:14<08:01, 2.00it/s]\u001b[A\n", + " 48%|███████████████████▍ | 904/1864 [07:15<07:42, 2.07it/s]\u001b[A\n", + " 49%|███████████████████▍ | 905/1864 [07:15<08:04, 1.98it/s]\u001b[A\n", + " 49%|███████████████████▍ | 906/1864 [07:16<07:49, 2.04it/s]\u001b[A\n", + " 49%|███████████████████▍ | 907/1864 [07:16<07:34, 2.11it/s]\u001b[A\n", + " 49%|███████████████████▍ | 908/1864 [07:17<07:42, 2.07it/s]\u001b[A\n", + " 49%|███████████████████▌ | 909/1864 [07:17<07:41, 2.07it/s]\u001b[A\n", + " 49%|███████████████████▌ | 910/1864 [07:18<07:07, 2.23it/s]\u001b[A\n", + " 49%|███████████████████▌ | 911/1864 [07:18<07:24, 2.14it/s]\u001b[A\n", + " 49%|███████████████████▌ | 912/1864 [07:19<07:24, 2.14it/s]\u001b[A\n", + " 49%|███████████████████▌ | 913/1864 [07:19<07:33, 2.10it/s]\u001b[A\n", + " 49%|███████████████████▌ | 914/1864 [07:20<08:06, 1.95it/s]\u001b[A\n", + " 49%|███████████████████▋ | 915/1864 [07:20<07:46, 2.04it/s]\u001b[A\n", + " 49%|███████████████████▋ | 916/1864 [07:21<07:29, 2.11it/s]\u001b[A\n", + " 49%|███████████████████▋ | 917/1864 [07:21<07:12, 2.19it/s]\u001b[A\n", + " 49%|███████████████████▋ | 918/1864 [07:22<07:30, 2.10it/s]\u001b[A\n", + " 49%|███████████████████▋ | 919/1864 [07:22<08:06, 1.94it/s]\u001b[A\n", + " 49%|███████████████████▋ | 920/1864 [07:23<07:44, 2.03it/s]\u001b[A\n", + " 49%|███████████████████▊ | 921/1864 [07:23<07:40, 2.05it/s]\u001b[A\n", + " 49%|███████████████████▊ | 922/1864 [07:24<07:42, 2.04it/s]\u001b[A\n", + " 50%|███████████████████▊ | 923/1864 [07:24<08:03, 1.95it/s]\u001b[A\n", + " 50%|███████████████████▊ | 924/1864 [07:25<07:48, 2.01it/s]\u001b[A\n", + " 50%|███████████████████▊ | 925/1864 [07:25<07:52, 1.99it/s]\u001b[A\n", + " 50%|███████████████████▊ | 926/1864 [07:26<08:28, 1.85it/s]\u001b[A\n", + " 50%|███████████████████▉ | 927/1864 [07:26<08:12, 1.90it/s]\u001b[A\n", + " 50%|███████████████████▉ | 928/1864 [07:27<08:05, 1.93it/s]\u001b[A\n", + " 50%|███��███████████████▉ | 929/1864 [07:27<08:23, 1.86it/s]\u001b[A\n", + " 50%|███████████████████▉ | 930/1864 [07:28<07:51, 1.98it/s]\u001b[A\n", + " 50%|███████████████████▉ | 931/1864 [07:28<07:38, 2.03it/s]\u001b[A\n", + " 50%|████████████████████ | 932/1864 [07:29<07:21, 2.11it/s]\u001b[A\n", + " 50%|████████████████████ | 933/1864 [07:29<07:47, 1.99it/s]\u001b[A\n", + " 50%|████████████████████ | 934/1864 [07:30<07:41, 2.01it/s]\u001b[A\n", + " 50%|████████████████████ | 935/1864 [07:30<07:44, 2.00it/s]\u001b[A\n", + " 50%|████████████████████ | 936/1864 [07:31<07:36, 2.03it/s]\u001b[A\n", + " 50%|████████████████████ | 937/1864 [07:31<07:56, 1.95it/s]\u001b[A\n", + " 50%|████████████████████▏ | 938/1864 [07:32<07:15, 2.13it/s]\u001b[A\n", + " 50%|████████████████████▏ | 939/1864 [07:32<07:16, 2.12it/s]\u001b[A\n", + " 50%|████████████████████▏ | 940/1864 [07:33<07:06, 2.17it/s]\u001b[A\n", + " 50%|████████████████████▏ | 941/1864 [07:33<07:41, 2.00it/s]\u001b[A\n", + " 51%|████████████████████▏ | 942/1864 [07:34<07:28, 2.05it/s]\u001b[A\n", + " 51%|████████████████████▏ | 943/1864 [07:34<07:19, 2.09it/s]\u001b[A\n", + " 51%|████████████████████▎ | 944/1864 [07:35<07:54, 1.94it/s]\u001b[A\n", + " 51%|████████████████████▎ | 945/1864 [07:35<08:00, 1.91it/s]\u001b[A\n", + " 51%|████████████████████▎ | 946/1864 [07:36<08:04, 1.89it/s]\u001b[A\n", + " 51%|████████████████████▎ | 947/1864 [07:36<07:50, 1.95it/s]\u001b[A\n", + " 51%|████████████████████▎ | 948/1864 [07:37<08:08, 1.87it/s]\u001b[A\n", + " 51%|████████████████████▎ | 949/1864 [07:37<07:59, 1.91it/s]\u001b[A\n", + " 51%|████████████████████▍ | 950/1864 [07:38<08:01, 1.90it/s]\u001b[A\n", + " 51%|████████████████████▍ | 951/1864 [07:38<08:21, 1.82it/s]\u001b[A\n", + " 51%|████████████████████▍ | 952/1864 [07:39<08:40, 1.75it/s]\u001b[A\n", + " 51%|████████████████████▍ | 953/1864 [07:39<08:06, 1.87it/s]\u001b[A\n", + " 51%|████████████████████▍ | 954/1864 [07:40<07:58, 1.90it/s]\u001b[A\n", + " 51%|████████████████████▍ | 955/1864 [07:40<07:23, 2.05it/s]\u001b[A\n", + " 51%|████████████████████▌ | 956/1864 [07:41<07:36, 1.99it/s]\u001b[A\n", + " 51%|████████████████████▌ | 957/1864 [07:42<08:05, 1.87it/s]\u001b[A\n", + " 51%|████████████████████▌ | 958/1864 [07:42<07:49, 1.93it/s]\u001b[A\n", + " 51%|████████████████████▌ | 959/1864 [07:42<07:08, 2.11it/s]\u001b[A\n", + " 52%|████████████████████▌ | 960/1864 [07:43<07:24, 2.03it/s]\u001b[A\n", + " 52%|████████████████████▌ | 961/1864 [07:43<07:07, 2.11it/s]\u001b[A\n", + " 52%|████████████████████▋ | 962/1864 [07:44<06:56, 2.16it/s]\u001b[A\n", + " 52%|████████████████████▋ | 963/1864 [07:44<07:09, 2.10it/s]\u001b[A\n", + " 52%|████████████████████▋ | 964/1864 [07:45<06:58, 2.15it/s]\u001b[A\n", + " 52%|���███████████████████▋ | 965/1864 [07:45<06:53, 2.17it/s]\u001b[A\n", + " 52%|████████████████████▋ | 966/1864 [07:46<06:35, 2.27it/s]\u001b[A\n", + " 52%|████████████████████▊ | 967/1864 [07:46<06:33, 2.28it/s]\u001b[A\n", + " 52%|████████████████████▊ | 968/1864 [07:46<06:20, 2.35it/s]\u001b[A\n", + " 52%|████████████████████▊ | 969/1864 [07:47<06:43, 2.22it/s]\u001b[A\n", + " 52%|████████████████████▊ | 970/1864 [07:47<06:51, 2.17it/s]\u001b[A\n", + " 52%|████████████████████▊ | 971/1864 [07:48<06:58, 2.13it/s]\u001b[A\n", + " 52%|████████████████████▊ | 972/1864 [07:48<06:55, 2.15it/s]\u001b[A\n", + " 52%|████████████████████▉ | 973/1864 [07:49<07:12, 2.06it/s]\u001b[A\n", + " 52%|████████████████████▉ | 974/1864 [07:49<07:31, 1.97it/s]\u001b[A\n", + " 52%|████████████████████▉ | 975/1864 [07:50<07:52, 1.88it/s]\u001b[A\n", + " 52%|████████████████████▉ | 976/1864 [07:50<07:14, 2.04it/s]\u001b[A\n", + " 52%|████████████████████▉ | 977/1864 [07:51<07:20, 2.01it/s]\u001b[A\n", + " 52%|████████████████████▉ | 978/1864 [07:51<07:16, 2.03it/s]\u001b[A\n", + " 53%|█████████████████████ | 979/1864 [07:52<07:11, 2.05it/s]\u001b[A\n", + " 53%|█████████████████████ | 980/1864 [07:52<07:18, 2.02it/s]\u001b[A\n", + " 53%|█████████████████████ | 981/1864 [07:53<07:02, 2.09it/s]\u001b[A\n", + " 53%|█████████████████████ | 982/1864 [07:53<07:19, 2.01it/s]\u001b[A\n", + " 53%|█████████████████████ | 983/1864 [07:54<07:03, 2.08it/s]\u001b[A\n", + " 53%|█████████████████████ | 984/1864 [07:54<06:58, 2.10it/s]\u001b[A\n", + " 53%|█████████████████████▏ | 985/1864 [07:55<07:00, 2.09it/s]\u001b[A\n", + " 53%|█████████████████████▏ | 986/1864 [07:55<06:54, 2.12it/s]\u001b[A\n", + " 53%|█████████████████████▏ | 987/1864 [07:56<07:08, 2.05it/s]\u001b[A\n", + " 53%|█████████████████████▏ | 988/1864 [07:56<07:10, 2.03it/s]\u001b[A\n", + " 53%|█████████████████████▏ | 989/1864 [07:57<07:08, 2.04it/s]\u001b[A\n", + " 53%|█████████████████████▏ | 990/1864 [07:57<07:08, 2.04it/s]\u001b[A\n", + " 53%|█████████████████████▎ | 991/1864 [07:58<06:49, 2.13it/s]\u001b[A\n", + " 53%|█████████████████████▎ | 992/1864 [07:58<06:47, 2.14it/s]\u001b[A\n", + " 53%|█████████████████████▎ | 993/1864 [07:59<07:15, 2.00it/s]\u001b[A\n", + " 53%|█████████████████████▎ | 994/1864 [07:59<07:21, 1.97it/s]\u001b[A\n", + " 53%|█████████████████████▎ | 995/1864 [08:00<07:01, 2.06it/s]\u001b[A\n", + " 53%|█████████████████████▎ | 996/1864 [08:00<06:51, 2.11it/s]\u001b[A\n", + " 53%|█████████████████████▍ | 997/1864 [08:01<06:42, 2.15it/s]\u001b[A\n", + " 54%|█████████████████████▍ | 998/1864 [08:01<07:08, 2.02it/s]\u001b[A\n", + " 54%|█████████████████████▍ | 999/1864 [08:02<07:07, 2.03it/s]\u001b[A\n", + " 54%|████████████████████▉ | 1000/1864 [08:02<07:01, 2.05it/s]\u001b[A\n", + " 54%|████████████████████▉ | 1001/1864 [08:03<07:02, 2.04it/s]\u001b[A\n", + " 54%|████████████████████▉ | 1002/1864 [08:03<07:00, 2.05it/s]\u001b[A\n", + " 54%|████████████████████▉ | 1003/1864 [08:04<06:57, 2.06it/s]\u001b[A\n", + " 54%|█████████████████████ | 1004/1864 [08:04<07:02, 2.03it/s]\u001b[A\n", + " 54%|█████████████████████ | 1005/1864 [08:05<07:02, 2.03it/s]\u001b[A\n", + " 54%|█████████████████████ | 1006/1864 [08:05<06:57, 2.06it/s]\u001b[A\n", + " 54%|█████████████████████ | 1007/1864 [08:06<07:09, 2.00it/s]\u001b[A\n", + " 54%|█████████████████████ | 1008/1864 [08:06<07:32, 1.89it/s]\u001b[A\n", + " 54%|█████████████████████ | 1009/1864 [08:07<07:27, 1.91it/s]\u001b[A\n", + " 54%|█████████████████████▏ | 1010/1864 [08:07<07:12, 1.98it/s]\u001b[A\n", + " 54%|█████████████████████▏ | 1011/1864 [08:08<07:02, 2.02it/s]\u001b[A\n", + " 54%|█████████████████████▏ | 1012/1864 [08:08<06:54, 2.05it/s]\u001b[A\n", + " 54%|█████████████████████▏ | 1013/1864 [08:09<07:23, 1.92it/s]\u001b[A\n", + " 54%|█████████████████████▏ | 1014/1864 [08:09<07:02, 2.01it/s]\u001b[A\n", + " 54%|█████████████████████▏ | 1015/1864 [08:10<07:00, 2.02it/s]\u001b[A\n", + " 55%|█████████████████████▎ | 1016/1864 [08:10<06:57, 2.03it/s]\u001b[A\n", + " 55%|█████████████████████▎ | 1017/1864 [08:11<06:59, 2.02it/s]\u001b[A\n", + " 55%|█████████████████████▎ | 1018/1864 [08:11<07:23, 1.91it/s]\u001b[A\n", + " 55%|█████████████████████▎ | 1019/1864 [08:12<06:50, 2.06it/s]\u001b[A\n", + " 55%|█████████████████████▎ | 1020/1864 [08:12<06:48, 2.07it/s]\u001b[A\n", + " 55%|█████████████████████▎ | 1021/1864 [08:13<07:15, 1.93it/s]\u001b[A\n", + " 55%|█████████████████████▍ | 1022/1864 [08:13<07:08, 1.96it/s]\u001b[A\n", + " 55%|█████████████████████▍ | 1023/1864 [08:14<06:55, 2.03it/s]\u001b[A\n", + " 55%|█████████████████████▍ | 1024/1864 [08:14<07:05, 1.97it/s]\u001b[A\n", + " 55%|█████████████████████▍ | 1025/1864 [08:15<06:39, 2.10it/s]\u001b[A\n", + " 55%|█████████████████████▍ | 1026/1864 [08:15<06:16, 2.23it/s]\u001b[A\n", + " 55%|█████████████████████▍ | 1027/1864 [08:16<06:52, 2.03it/s]\u001b[A\n", + " 55%|█████████████████████▌ | 1028/1864 [08:16<06:56, 2.01it/s]\u001b[A\n", + " 55%|█████████████████████▌ | 1029/1864 [08:17<06:56, 2.01it/s]\u001b[A\n", + " 55%|█████████████████████▌ | 1030/1864 [08:17<06:55, 2.01it/s]\u001b[A\n", + " 55%|█████████████████████▌ | 1031/1864 [08:17<06:48, 2.04it/s]\u001b[A\n", + " 55%|█████████████████████▌ | 1032/1864 [08:18<07:07, 1.95it/s]\u001b[A\n", + " 55%|█████████████████████▌ | 1033/1864 [08:19<07:13, 1.92it/s]\u001b[A\n", + " 55%|█████████████████████▋ | 1034/1864 [08:19<07:12, 1.92it/s]\u001b[A\n", + " 56%|█████████████████████▋ | 1035/1864 [08:20<06:47, 2.04it/s]\u001b[A\n", + " 56%|█████████████████████▋ | 1036/1864 [08:20<06:44, 2.05it/s]\u001b[A\n", + " 56%|█████████████████████▋ | 1037/1864 [08:21<06:56, 1.99it/s]\u001b[A\n", + " 56%|█████████████████████▋ | 1038/1864 [08:21<06:41, 2.06it/s]\u001b[A\n", + " 56%|█████████████████████▋ | 1039/1864 [08:22<07:16, 1.89it/s]\u001b[A\n", + " 56%|█████████████████████▊ | 1040/1864 [08:22<07:28, 1.84it/s]\u001b[A\n", + " 56%|█████████████████████▊ | 1041/1864 [08:23<07:18, 1.88it/s]\u001b[A\n", + " 56%|█████████████████████▊ | 1042/1864 [08:23<07:19, 1.87it/s]\u001b[A\n", + " 56%|█████████████████████▊ | 1043/1864 [08:24<07:11, 1.90it/s]\u001b[A\n", + " 56%|█████████████████████▊ | 1044/1864 [08:24<06:35, 2.07it/s]\u001b[A\n", + " 56%|█████████████████████▊ | 1045/1864 [08:25<06:51, 1.99it/s]\u001b[A\n", + " 56%|█████████████████████▉ | 1046/1864 [08:25<06:38, 2.05it/s]\u001b[A\n", + " 56%|█████████████████████▉ | 1047/1864 [08:26<07:00, 1.94it/s]\u001b[A\n", + " 56%|█████████████████████▉ | 1048/1864 [08:26<06:50, 1.99it/s]\u001b[A\n", + " 56%|█████████████████████▉ | 1049/1864 [08:27<06:26, 2.11it/s]\u001b[A\n", + " 56%|█████████████████████▉ | 1050/1864 [08:27<06:45, 2.01it/s]\u001b[A\n", + " 56%|█████████████████████▉ | 1051/1864 [08:28<06:57, 1.95it/s]\u001b[A\n", + " 56%|██████████████████████ | 1052/1864 [08:28<07:31, 1.80it/s]\u001b[A\n", + " 56%|██████████████████████ | 1053/1864 [08:29<07:25, 1.82it/s]\u001b[A\n", + " 57%|██████████████████████ | 1054/1864 [08:29<07:25, 1.82it/s]\u001b[A\n", + " 57%|██████████████████████ | 1055/1864 [08:30<07:07, 1.89it/s]\u001b[A\n", + " 57%|██████████████████████ | 1056/1864 [08:30<06:57, 1.93it/s]\u001b[A\n", + " 57%|██████████████████████ | 1057/1864 [08:31<06:57, 1.93it/s]\u001b[A\n", + " 57%|██████████████████████▏ | 1058/1864 [08:31<06:28, 2.07it/s]\u001b[A\n", + " 57%|██████████████████████▏ | 1059/1864 [08:32<06:40, 2.01it/s]\u001b[A\n", + " 57%|██████████████████████▏ | 1060/1864 [08:32<06:30, 2.06it/s]\u001b[A\n", + " 57%|██████████████████████▏ | 1061/1864 [08:33<06:10, 2.17it/s]\u001b[A\n", + " 57%|██████████████████████▏ | 1062/1864 [08:33<05:57, 2.24it/s]\u001b[A\n", + " 57%|██████████████████████▏ | 1063/1864 [08:34<06:16, 2.13it/s]\u001b[A\n", + " 57%|██████████████████████▎ | 1064/1864 [08:34<06:13, 2.14it/s]\u001b[A\n", + " 57%|██████████████████████▎ | 1065/1864 [08:35<06:32, 2.04it/s]\u001b[A\n", + " 57%|██████████████████████▎ | 1066/1864 [08:35<06:18, 2.11it/s]\u001b[A\n", + " 57%|██████████████████████▎ | 1067/1864 [08:36<06:20, 2.10it/s]\u001b[A\n", + " 57%|██████████████████████▎ | 1068/1864 [08:36<06:17, 2.11it/s]\u001b[A\n", + " 57%|██████████████████████▎ | 1069/1864 [08:37<06:29, 2.04it/s]\u001b[A\n", + " 57%|██████████████████████▍ | 1070/1864 [08:37<06:15, 2.11it/s]\u001b[A\n", + " 57%|████████���█████████████▍ | 1071/1864 [08:38<06:19, 2.09it/s]\u001b[A\n", + " 58%|██████████████████████▍ | 1072/1864 [08:38<05:55, 2.23it/s]\u001b[A\n", + " 58%|██████████████████████▍ | 1073/1864 [08:38<06:12, 2.13it/s]\u001b[A\n", + " 58%|██████████████████████▍ | 1074/1864 [08:39<06:22, 2.06it/s]\u001b[A\n", + " 58%|██████████████████████▍ | 1075/1864 [08:40<06:50, 1.92it/s]\u001b[A\n", + " 58%|██████████████████████▌ | 1076/1864 [08:40<06:55, 1.90it/s]\u001b[A\n", + " 58%|██████████████████████▌ | 1077/1864 [08:41<07:05, 1.85it/s]\u001b[A\n", + " 58%|██████████████████████▌ | 1078/1864 [08:41<07:05, 1.85it/s]\u001b[A\n", + " 58%|██████████████████████▌ | 1079/1864 [08:42<06:52, 1.90it/s]\u001b[A\n", + " 58%|██████████████████████▌ | 1080/1864 [08:42<06:55, 1.89it/s]\u001b[A\n", + " 58%|██████████████████████▌ | 1081/1864 [08:43<06:45, 1.93it/s]\u001b[A\n", + " 58%|██████████████████████▋ | 1082/1864 [08:43<06:35, 1.98it/s]\u001b[A\n", + " 58%|██████████████████████▋ | 1083/1864 [08:44<06:02, 2.16it/s]\u001b[A\n", + " 58%|██████████████████████▋ | 1084/1864 [08:44<06:06, 2.13it/s]\u001b[A\n", + " 58%|██████████████████████▋ | 1085/1864 [08:45<06:11, 2.09it/s]\u001b[A\n", + " 58%|██████████████████████▋ | 1086/1864 [08:45<06:19, 2.05it/s]\u001b[A\n", + " 58%|██████████████████████▋ | 1087/1864 [08:46<06:13, 2.08it/s]\u001b[A\n", + " 58%|██████████████████████▊ | 1088/1864 [08:46<05:55, 2.18it/s]\u001b[A\n", + " 58%|██████████████████████▊ | 1089/1864 [08:46<06:19, 2.04it/s]\u001b[A\n", + " 58%|██████████████████████▊ | 1090/1864 [08:47<06:11, 2.08it/s]\u001b[A\n", + " 59%|██████████████████████▊ | 1091/1864 [08:48<06:36, 1.95it/s]\u001b[A\n", + " 59%|██████████████████████▊ | 1092/1864 [08:48<06:33, 1.96it/s]\u001b[A\n", + " 59%|██████████████████████▊ | 1093/1864 [08:49<06:37, 1.94it/s]\u001b[A\n", + " 59%|██████████████████████▉ | 1094/1864 [08:49<06:32, 1.96it/s]\u001b[A\n", + " 59%|██████████████████████▉ | 1095/1864 [08:49<06:12, 2.06it/s]\u001b[A\n", + " 59%|██████████████████████▉ | 1096/1864 [08:50<06:10, 2.07it/s]\u001b[A\n", + " 59%|██████████████████████▉ | 1097/1864 [08:51<06:25, 1.99it/s]\u001b[A\n", + " 59%|██████████████████████▉ | 1098/1864 [08:51<06:02, 2.12it/s]\u001b[A\n", + " 59%|██████████████████████▉ | 1099/1864 [08:51<05:52, 2.17it/s]\u001b[A\n", + " 59%|███████████████████████ | 1100/1864 [08:52<06:00, 2.12it/s]\u001b[A\n", + " 59%|███████████████████████ | 1101/1864 [08:52<05:59, 2.12it/s]\u001b[A\n", + " 59%|███████████████████████ | 1102/1864 [08:53<06:01, 2.11it/s]\u001b[A\n", + " 59%|███████████████████████ | 1103/1864 [08:53<05:44, 2.21it/s]\u001b[A\n", + " 59%|███████████████████████ | 1104/1864 [08:54<06:02, 2.10it/s]\u001b[A\n", + " 59%|███████████████████████ | 1105/1864 [08:54<06:07, 2.06it/s]\u001b[A\n", + " 59%|███████████████████████▏ | 1106/1864 [08:55<06:07, 2.06it/s]\u001b[A\n", + " 59%|███████████████████████▏ | 1107/1864 [08:55<06:32, 1.93it/s]\u001b[A\n", + " 59%|███████████████████████▏ | 1108/1864 [08:56<06:58, 1.81it/s]\u001b[A\n", + " 59%|███████████████████████▏ | 1109/1864 [08:57<06:59, 1.80it/s]\u001b[A\n", + " 60%|███████████████████████▏ | 1110/1864 [08:57<06:48, 1.85it/s]\u001b[A\n", + " 60%|███████████████████████▏ | 1111/1864 [08:57<06:34, 1.91it/s]\u001b[A\n", + " 60%|███████████████████████▎ | 1112/1864 [08:58<06:49, 1.84it/s]\u001b[A\n", + " 60%|███████████████████████▎ | 1113/1864 [08:59<07:00, 1.79it/s]\u001b[A\n", + " 60%|███████████████████████▎ | 1114/1864 [08:59<07:09, 1.75it/s]\u001b[A\n", + " 60%|███████████████████████▎ | 1115/1864 [09:00<07:06, 1.75it/s]\u001b[A\n", + " 60%|███████████████████████▎ | 1116/1864 [09:00<07:17, 1.71it/s]\u001b[A\n", + " 60%|███████████████████████▎ | 1117/1864 [09:01<06:50, 1.82it/s]\u001b[A\n", + " 60%|███████████████████████▍ | 1118/1864 [09:02<07:01, 1.77it/s]\u001b[A\n", + " 60%|███████████████████████▍ | 1119/1864 [09:02<06:47, 1.83it/s]\u001b[A\n", + " 60%|███████████████████████▍ | 1120/1864 [09:03<06:35, 1.88it/s]\u001b[A\n", + " 60%|███████████████████████▍ | 1121/1864 [09:03<06:28, 1.91it/s]\u001b[A\n", + " 60%|███████████████████████▍ | 1122/1864 [09:04<06:14, 1.98it/s]\u001b[A\n", + " 60%|███████████████████████▍ | 1123/1864 [09:04<06:20, 1.95it/s]\u001b[A\n", + " 60%|███████████████████████▌ | 1124/1864 [09:05<06:20, 1.95it/s]\u001b[A\n", + " 60%|███████████████████████▌ | 1125/1864 [09:05<06:39, 1.85it/s]\u001b[A\n", + " 60%|███████████████████████▌ | 1126/1864 [09:06<06:31, 1.88it/s]\u001b[A\n", + " 60%|███████████████████████▌ | 1127/1864 [09:06<06:10, 1.99it/s]\u001b[A\n", + " 61%|███████████████████████▌ | 1128/1864 [09:06<05:46, 2.13it/s]\u001b[A\n", + " 61%|███████████████████████▌ | 1129/1864 [09:07<06:03, 2.02it/s]\u001b[A\n", + " 61%|███████████████████████▋ | 1130/1864 [09:08<06:08, 1.99it/s]\u001b[A\n", + " 61%|███████████████████████▋ | 1131/1864 [09:08<06:06, 2.00it/s]\u001b[A\n", + " 61%|███████████████████████▋ | 1132/1864 [09:09<06:20, 1.92it/s]\u001b[A\n", + " 61%|███████████████████████▋ | 1133/1864 [09:09<06:17, 1.94it/s]\u001b[A\n", + " 61%|███████████████████████▋ | 1134/1864 [09:10<06:00, 2.03it/s]\u001b[A\n", + " 61%|███████████████████████▋ | 1135/1864 [09:10<06:01, 2.02it/s]\u001b[A\n", + " 61%|███████████████████████▊ | 1136/1864 [09:10<05:31, 2.20it/s]\u001b[A\n", + " 61%|███████████████████████▊ | 1137/1864 [09:11<05:33, 2.18it/s]\u001b[A\n", + " 61%|███████████████████████▊ | 1138/1864 [09:11<06:01, 2.01it/s]\u001b[A\n", + " 61%|███████████████████████▊ | 1139/1864 [09:12<05:45, 2.10it/s]\u001b[A\n", + " 61%|███████████████████████▊ | 1140/1864 [09:12<05:46, 2.09it/s]\u001b[A\n", + " 61%|███████████████████████▊ | 1141/1864 [09:13<05:58, 2.02it/s]\u001b[A\n", + " 61%|███████████████████████▉ | 1142/1864 [09:14<06:19, 1.90it/s]\u001b[A\n", + " 61%|███████████████████████▉ | 1143/1864 [09:14<06:08, 1.96it/s]\u001b[A\n", + " 61%|███████████████████████▉ | 1144/1864 [09:14<05:48, 2.06it/s]\u001b[A\n", + " 61%|███████████████████████▉ | 1145/1864 [09:15<05:54, 2.03it/s]\u001b[A\n", + " 61%|███████████████████████▉ | 1146/1864 [09:15<05:46, 2.07it/s]\u001b[A\n", + " 62%|███████████████████████▉ | 1147/1864 [09:16<05:21, 2.23it/s]\u001b[A\n", + " 62%|████████████████████████ | 1148/1864 [09:16<05:04, 2.35it/s]\u001b[A\n", + " 62%|████████████████████████ | 1149/1864 [09:17<05:21, 2.23it/s]\u001b[A\n", + " 62%|████████████████████████ | 1150/1864 [09:17<05:45, 2.06it/s]\u001b[A\n", + " 62%|████████████████████████ | 1151/1864 [09:18<05:33, 2.13it/s]\u001b[A\n", + " 62%|████████████████████████ | 1152/1864 [09:18<05:27, 2.17it/s]\u001b[A\n", + " 62%|████████████████████████ | 1153/1864 [09:19<05:17, 2.24it/s]\u001b[A\n", + " 62%|████████████████████████▏ | 1154/1864 [09:19<05:26, 2.17it/s]\u001b[A\n", + " 62%|████████████████████████▏ | 1155/1864 [09:19<05:16, 2.24it/s]\u001b[A\n", + " 62%|████████████████████████▏ | 1156/1864 [09:20<05:42, 2.07it/s]\u001b[A\n", + " 62%|████████████████████████▏ | 1157/1864 [09:20<05:43, 2.06it/s]\u001b[A\n", + " 62%|████████████████████████▏ | 1158/1864 [09:21<05:01, 2.34it/s]\u001b[A\n", + " 62%|████████████████████████▏ | 1159/1864 [09:21<05:35, 2.10it/s]\u001b[A\n", + " 62%|████████████████████████▎ | 1160/1864 [09:22<05:35, 2.10it/s]\u001b[A\n", + " 62%|████████████████████████▎ | 1161/1864 [09:22<05:46, 2.03it/s]\u001b[A\n", + " 62%|████████████████████████▎ | 1162/1864 [09:23<05:50, 2.00it/s]\u001b[A\n", + " 62%|████████████████████████▎ | 1163/1864 [09:23<05:54, 1.98it/s]\u001b[A\n", + " 62%|████████████████████████▎ | 1164/1864 [09:24<06:02, 1.93it/s]\u001b[A\n", + " 62%|████████████████████████▍ | 1165/1864 [09:24<05:58, 1.95it/s]\u001b[ADeleting older checkpoint [checkpoint-3500] due to args.save_total_limit\n", + "{'loss': 1.0462, 'learning_rate': 7.405038838920686e-05, 'epoch': 1.03} \n", + "{'loss': 1.0562, 'learning_rate': 7.401972608340146e-05, 'epoch': 1.05} \n", + "{'loss': 1.0388, 'learning_rate': 7.398906377759606e-05, 'epoch': 1.07} \n", + "{'loss': 1.056, 'learning_rate': 7.395840147179068e-05, 'epoch': 1.09} \n", + "{'loss': 1.0459, 'learning_rate': 7.392773916598528e-05, 'epoch': 1.12} \n", + " 2%|▋ | 5500/246600 [5:31:16<147:16:22, 2.20s/it]The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 14905\n", + " Batch size = 8\n", + "\n", + " 0%| | 0/1864 [00:00