Saving train state of step 5

distil-whisper/events.out.tfevents.1715173957.server02.1931183.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0baaf5c51a7181547abed27026c4d96f6e67c79735b12da3ff8ab92b5ad8d34
+size 88

distil-whisper/events.out.tfevents.1715174400.server02.1934277.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d55b57f53ba824e31a3e70e10be7b81fffbc7136cf863b03721bb34f860c36e1
+size 88

distil-whisper/events.out.tfevents.1715174461.server02.1934867.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a7cbaa5531dd3b4ce37f609a4f68d565dbcdc01c5b996dc862d7c531efbe032
+size 88

distil-whisper/events.out.tfevents.1715174772.server02.1937015.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17d80187904c3f499e62daa25bd6a595704610866a8ca14f7464f436cae37096
+size 88

distil-whisper/events.out.tfevents.1715174837.server02.1937715.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba268dc4a701175857d00571487bd02a56b657427e5c6c19bbc4d5d828ee2479
+size 88

distil-whisper/events.out.tfevents.1715174907.server02.1938409.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b99ee4c468275a20eec36de30b0e0bd3326c2605623203ae6724843aea1dfc78
+size 88

distil-whisper/events.out.tfevents.1715183755.server02.1990428.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89067edc26fa465bd32a850c8f10ead9195d7e28ea74ffab40e7e4c485c2e403
+size 88

run_distillation.py

@@ -1312,7 +1312,8 @@ def main():
         num_epochs = int(training_args.num_train_epochs)
         steps_per_epoch = len(vectorized_datasets["train"]) // (train_batch_size * gradient_accumulation_steps)
         total_train_steps = steps_per_epoch * num_epochs
-    elif training_args.max_steps > 0:
+
+    elif training_args.max_steps > 0: #since we use data streaming , this condition is satisfied
         logger.info("max_steps is given, it will override any value given in num_train_epochs")
         total_train_steps = int(training_args.max_steps)
         if not data_args.streaming:
@@ -1427,14 +1428,14 @@ def main():
         student_model.train()
         teacher_model.eval()
 
-        student_outputs = student_model(**batch)
+        student_outputs = student_model(**batch) # __call__ is overidden for forward function , note : student_model and teacher model both are whisperforconditionalgeneration object
         with torch.no_grad():
-            if share_hidden_states:
+            if share_hidden_states: 
                 # if the student and teacher share the same frozen encoder then we don't have to recompute the
                 # encoder hidden-states for the teacher model, we can just re-use from the student
                 encoder_outputs = BaseModelOutput(student_outputs.encoder_last_hidden_state.to(dtype=teacher_dtype))
                 teacher_outputs = teacher_model(encoder_outputs=encoder_outputs, labels=batch["labels"])
-            else:
+            else: 
                 # do the full forward pass for the teacher model (encoder + decoder)
                 teacher_outputs = teacher_model(**batch) 
           
@@ -1546,8 +1547,24 @@ def main():
     print(f" vectorized_datasets['eval'] : {vectorized_datasets['eval']}")
     print(f" vectorized_datasets['train'] : {vectorized_datasets['train']}")
 
-    
+    #see example of validation dataloader
+    # validation_dataloader = DataLoader(  
+    #                         vectorized_datasets[eval_split],  
+    #                         collate_fn=data_collator,   
+    #                         batch_size=per_device_eval_batch_size, 
+    #                         drop_last=False, 
+    #                         num_workers=dataloader_num_workers, 
+    #                         prefetch_factor=prefetch_factor, 
+    #                         pin_memory=training_args.dataloader_pin_memory,
+    #                     )
+
+    # for batch in validation_dataloader:
+    #     print(batch['input_features'].shape) 
+        
 
+    print(f" student_model : {type(student_model)}")
+    
+    
     for epoch in range(epochs_trained, num_epochs):
         vectorized_datasets["train"] = vectorized_datasets["train"].shuffle(training_args.seed)
         train_dataloader = DataLoader(
@@ -1570,10 +1587,13 @@ def main():
         
         for batch in train_dataloader:
             with accelerator.accumulate(student_model):
+                #they are updated their parameters every batch
                 loss, train_metric = train_step(batch, temperature=training_args.temperature)
+                #backward pass with loss
                 accelerator.backward(loss)
                 if accelerator.sync_gradients:
                     accelerator.clip_grad_norm_(student_model.parameters(), training_args.max_grad_norm)
+                #update after forward method
                 optimizer.step()
                 lr_scheduler.step()
                 optimizer.zero_grad()
@@ -1582,7 +1602,9 @@ def main():
             if accelerator.sync_gradients:
                 steps_trained_progress_bar.update(1)
                 cur_step += 1
+                
 
+                #logging timing 
                 if cur_step % training_args.logging_steps == 0:
                     steps_trained_progress_bar.write(
                         f"Step... ({cur_step} / {total_train_steps} | Loss:"
@@ -1733,5 +1755,5 @@ def main():
 if __name__ == "__main__":
     main()
     '''
-    accelerate launch --mixed_precision=bf16 run_distillation.py   --model_name_or_path "./distil-large-v3-init"   --teacher_model_name_or_path "openai/whisper-large-v3"   --train_dataset_name "mozilla-foundation/common_voice_15_0"   --train_dataset_config_name "de"   --train_split_name "train"   --text_column_name "sentence"   --eval_dataset_name "mozilla-foundation/common_voice_15_0"   --eval_dataset_config_name "de"   --eval_split_name "validation"   --eval_text_column_name "sentence"   --eval_steps 5 --save_steps 50   --warmup_steps 500   --learning_rate 1e-4   --lr_scheduler_type "linear"   --logging_steps 25   --save_total_limit 1   --max_steps 5   --per_device_train_batch_size 4   --per_device_eval_batch_size 2   --dataloader_num_workers 2   --preprocessing_num_workers 2   --ddp_timeout 7200   --dtype "bfloat16"   --output_dir "./"   --use_pseudo_labels "false"   --condition_on_prev_probability "0.0"   --do_train   --do_eval   --gradient_checkpointing   --overwrite_output_dir   --predict_with_generate   --freeze_encoder   --streaming   --push_to_hub --language de
+    accelerate launch --mixed_precision=bf16 run_distillation.py   --model_name_or_path "./distil-large-v3-init"   --teacher_model_name_or_path "openai/whisper-large-v3"   --train_dataset_name "mozilla-foundation/common_voice_15_0"   --train_dataset_config_name "de"   --train_split_name "train"   --text_column_name "sentence"   --eval_dataset_name "mozilla-foundation/common_voice_15_0"   --eval_dataset_config_name "de"   --eval_split_name "validation"   --eval_text_column_name "sentence"   --eval_steps 5 --save_steps 50   --warmup_steps 500   --learning_rate 1e-4   --lr_scheduler_type "linear"   --logging_steps 25   --save_total_limit 1   --max_steps 5   --per_device_train_batch_size 4   --per_device_eval_batch_size 2   --dataloader_num_workers 2   --preprocessing_num_workers 2   --ddp_timeout 7200   --dtype "bfloat16"   --output_dir "./"   --use_pseudo_labels "false"   --condition_on_prev_probability "0.0"   --do_train   --do_eval   --gradient_checkpointing   --overwrite_output_dir   --predict_with_generate   --freeze_encoder   --streaming   --push_to_hub --language de --max_eval_samples 5
     '''