phi-3.5-new

adapter_config.json

@@ -12,9 +12,9 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 32,
+  "lora_alpha": 64,
   "lora_bias": false,
-  "lora_dropout": 0.05,
+  "lora_dropout": 0.1,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
@@ -23,10 +23,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "down_proj",
     "qkv_proj",
-    "gate_up_proj",
     "o_proj",
-    "down_proj"
+    "gate_up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20178576bc8573e56eb14493ece1b59f3ae774a58a2e0780f176f7c0d3a70c29
+oid sha256:41b60cffda93a140952b4e8e313fd8a8519ca094dd39b7008a23a9942e80caa3
 size 100697728

all_results.json

@@ -1,12 +1,12 @@
 {
-    "epoch": 20.0,
-    "eval_loss": 0.28774118423461914,
-    "eval_runtime": 16.9372,
-    "eval_samples_per_second": 2.48,
-    "eval_steps_per_second": 1.24,
-    "total_flos": 8.68297895581778e+16,
-    "train_loss": 0.616850325694451,
-    "train_runtime": 5601.927,
-    "train_samples_per_second": 0.743,
-    "train_steps_per_second": 0.371
+    "epoch": 7.0,
+    "eval_loss": 1.0098934173583984,
+    "eval_runtime": 7.0732,
+    "eval_samples_per_second": 2.403,
+    "eval_steps_per_second": 1.272,
+    "total_flos": 1.2128346681348096e+16,
+    "train_loss": 0.8996787428044949,
+    "train_runtime": 846.9703,
+    "train_samples_per_second": 0.98,
+    "train_steps_per_second": 0.496
 }

eval_results.json

@@ -1,7 +1,7 @@
 {
-    "epoch": 20.0,
-    "eval_loss": 0.28774118423461914,
-    "eval_runtime": 16.9372,
-    "eval_samples_per_second": 2.48,
-    "eval_steps_per_second": 1.24
+    "epoch": 7.0,
+    "eval_loss": 1.0098934173583984,
+    "eval_runtime": 7.0732,
+    "eval_samples_per_second": 2.403,
+    "eval_steps_per_second": 1.272
 }

runs/Jan12_18-16-10_dmlab/events.out.tfevents.1736698571.dmlab.4621.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d54c2a3766204553f23b25a8cf1a7a3efa310798eff0e961e6d46a1667e7ba00
+size 8519

runs/Jan12_18-39-23_dmlab/events.out.tfevents.1736699964.dmlab.4828.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd10c8db7c1dedcc2d3774aeeb9feec748f2e7d0719a944e669bfb18eb98ea66
+size 8310

runs/Jan12_18-40-28_dmlab/events.out.tfevents.1736700029.dmlab.5135.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba41872b0a2f5a89c3b52534b7ef19d53cfc00f43a4083cab9dc00cbd774f61d
+size 8313

runs/Jan12_18-48-53_dmlab/events.out.tfevents.1736700534.dmlab.6375.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3497c533857fddcc00a15e1c8cde06464b3e72e93550f9cf03f1d0553bdd6111
+size 8312

runs/Jan12_18-49-27_dmlab/events.out.tfevents.1736700567.dmlab.6681.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ceb695ef6ca804bf37ca664d0a33b0e9b6ec1ad7bc5372f866735be62248c45a
+size 8312

runs/Jan12_18-49-56_dmlab/events.out.tfevents.1736700596.dmlab.6967.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8bc81128a4ec26f0f4aaeb977cbbd7e8ef5f8b3eb0cff993670330e67af50c2
+size 12024

runs/Jan12_18-49-56_dmlab/events.out.tfevents.1736701450.dmlab.6967.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61be2c6a36537aba58fb650dbf5d51eb1275adbe01d9e83dc4c273c62e5972d9
+size 359

tokenizer_config.json

@@ -121,7 +121,7 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,
-  "model_max_length": 2048,
+  "model_max_length": 4096,
   "pad_token": "<unk>",
   "padding_side": "left",
   "sp_model_kwargs": {},

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 20.0,
-    "total_flos": 8.68297895581778e+16,
-    "train_loss": 0.616850325694451,
-    "train_runtime": 5601.927,
-    "train_samples_per_second": 0.743,
-    "train_steps_per_second": 0.371
+    "epoch": 7.0,
+    "total_flos": 1.2128346681348096e+16,
+    "train_loss": 0.8996787428044949,
+    "train_runtime": 846.9703,
+    "train_samples_per_second": 0.98,
+    "train_steps_per_second": 0.496
 }

trainer_state.json

@@ -1,169 +1,143 @@
 {
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 20.0,
+  "best_metric": 1.0029878616333008,
+  "best_model_checkpoint": "/home/labuser/Documents/phi-3/phi-3.5-new/checkpoint-168",
+  "epoch": 7.0,
   "eval_steps": 500,
-  "global_step": 2080,
+  "global_step": 294,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 1.0,
-      "grad_norm": 0.6938191056251526,
-      "learning_rate": 5e-05,
-      "loss": 1.1644,
-      "step": 104
-    },
-    {
-      "epoch": 2.0,
-      "grad_norm": 0.5342716574668884,
+      "grad_norm": 1.0487970113754272,
       "learning_rate": 0.0001,
-      "loss": 0.9083,
-      "step": 208
+      "loss": 1.1905,
+      "step": 42
     },
     {
-      "epoch": 3.0,
-      "grad_norm": 0.47485536336898804,
-      "learning_rate": 0.00015000000000000001,
-      "loss": 0.8697,
-      "step": 312
+      "epoch": 1.0,
+      "eval_loss": 1.0386497974395752,
+      "eval_runtime": 7.0656,
+      "eval_samples_per_second": 2.406,
+      "eval_steps_per_second": 1.274,
+      "step": 42
     },
     {
-      "epoch": 4.0,
-      "grad_norm": 0.5520597100257874,
+      "epoch": 2.0,
+      "grad_norm": 1.0445537567138672,
       "learning_rate": 0.0002,
-      "loss": 0.8504,
-      "step": 416
+      "loss": 0.9483,
+      "step": 84
     },
     {
-      "epoch": 5.0,
-      "grad_norm": 0.4108010530471802,
-      "learning_rate": 0.00019807852804032305,
-      "loss": 0.8314,
-      "step": 520
+      "epoch": 2.0,
+      "eval_loss": 1.0180531740188599,
+      "eval_runtime": 7.0935,
+      "eval_samples_per_second": 2.397,
+      "eval_steps_per_second": 1.269,
+      "step": 84
     },
     {
-      "epoch": 6.0,
-      "grad_norm": 0.5089908838272095,
+      "epoch": 3.0,
+      "grad_norm": 0.8360362648963928,
       "learning_rate": 0.0001923879532511287,
-      "loss": 0.809,
-      "step": 624
+      "loss": 0.9012,
+      "step": 126
     },
     {
-      "epoch": 7.0,
-      "grad_norm": 0.5841424465179443,
-      "learning_rate": 0.00018314696123025454,
-      "loss": 0.788,
-      "step": 728
+      "epoch": 3.0,
+      "eval_loss": 1.0081464052200317,
+      "eval_runtime": 7.1072,
+      "eval_samples_per_second": 2.392,
+      "eval_steps_per_second": 1.266,
+      "step": 126
     },
     {
-      "epoch": 8.0,
-      "grad_norm": 0.6074467301368713,
+      "epoch": 4.0,
+      "grad_norm": 0.9580802917480469,
       "learning_rate": 0.00017071067811865476,
-      "loss": 0.7642,
-      "step": 832
+      "loss": 0.8626,
+      "step": 168
     },
     {
-      "epoch": 9.0,
-      "grad_norm": 0.6272623538970947,
-      "learning_rate": 0.00015555702330196023,
-      "loss": 0.7376,
-      "step": 936
+      "epoch": 4.0,
+      "eval_loss": 1.0029878616333008,
+      "eval_runtime": 7.081,
+      "eval_samples_per_second": 2.401,
+      "eval_steps_per_second": 1.271,
+      "step": 168
     },
     {
-      "epoch": 10.0,
-      "grad_norm": 0.6662757992744446,
+      "epoch": 5.0,
+      "grad_norm": 0.8999230861663818,
       "learning_rate": 0.000138268343236509,
-      "loss": 0.7044,
-      "step": 1040
+      "loss": 0.8324,
+      "step": 210
     },
     {
-      "epoch": 11.0,
-      "grad_norm": 0.9177679419517517,
-      "learning_rate": 0.00011950903220161285,
-      "loss": 0.6638,
-      "step": 1144
+      "epoch": 5.0,
+      "eval_loss": 1.0067561864852905,
+      "eval_runtime": 7.0886,
+      "eval_samples_per_second": 2.398,
+      "eval_steps_per_second": 1.27,
+      "step": 210
     },
     {
-      "epoch": 12.0,
-      "grad_norm": 1.0447787046432495,
+      "epoch": 6.0,
+      "grad_norm": 0.8453378081321716,
       "learning_rate": 0.0001,
-      "loss": 0.6131,
-      "step": 1248
+      "loss": 0.8004,
+      "step": 252
     },
     {
-      "epoch": 13.0,
-      "grad_norm": 1.1840310096740723,
-      "learning_rate": 8.049096779838719e-05,
-      "loss": 0.5513,
-      "step": 1352
+      "epoch": 6.0,
+      "eval_loss": 1.0179213285446167,
+      "eval_runtime": 7.0766,
+      "eval_samples_per_second": 2.402,
+      "eval_steps_per_second": 1.272,
+      "step": 252
     },
     {
-      "epoch": 14.0,
-      "grad_norm": 1.5581327676773071,
+      "epoch": 7.0,
+      "grad_norm": 1.0025222301483154,
       "learning_rate": 6.173165676349103e-05,
-      "loss": 0.4789,
-      "step": 1456
+      "loss": 0.7622,
+      "step": 294
     },
     {
-      "epoch": 15.0,
-      "grad_norm": 1.5830806493759155,
-      "learning_rate": 4.444297669803981e-05,
-      "loss": 0.4021,
-      "step": 1560
-    },
-    {
-      "epoch": 16.0,
-      "grad_norm": 1.7810040712356567,
-      "learning_rate": 2.9289321881345254e-05,
-      "loss": 0.3294,
-      "step": 1664
-    },
-    {
-      "epoch": 17.0,
-      "grad_norm": 1.415872573852539,
-      "learning_rate": 1.6853038769745467e-05,
-      "loss": 0.2672,
-      "step": 1768
-    },
-    {
-      "epoch": 18.0,
-      "grad_norm": 1.7194544076919556,
-      "learning_rate": 7.612046748871327e-06,
-      "loss": 0.2226,
-      "step": 1872
-    },
-    {
-      "epoch": 19.0,
-      "grad_norm": 1.4402439594268799,
-      "learning_rate": 1.921471959676957e-06,
-      "loss": 0.1961,
-      "step": 1976
-    },
-    {
-      "epoch": 20.0,
-      "grad_norm": 1.3084591627120972,
-      "learning_rate": 0.0,
-      "loss": 0.1851,
-      "step": 2080
+      "epoch": 7.0,
+      "eval_loss": 1.043684959411621,
+      "eval_runtime": 7.0812,
+      "eval_samples_per_second": 2.401,
+      "eval_steps_per_second": 1.271,
+      "step": 294
     },
     {
-      "epoch": 20.0,
-      "step": 2080,
-      "total_flos": 8.68297895581778e+16,
-      "train_loss": 0.616850325694451,
-      "train_runtime": 5601.927,
-      "train_samples_per_second": 0.743,
-      "train_steps_per_second": 0.371
+      "epoch": 7.0,
+      "step": 294,
+      "total_flos": 1.2128346681348096e+16,
+      "train_loss": 0.8996787428044949,
+      "train_runtime": 846.9703,
+      "train_samples_per_second": 0.98,
+      "train_steps_per_second": 0.496
     }
   ],
   "logging_steps": 500,
-  "max_steps": 2080,
+  "max_steps": 420,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 20,
+  "num_train_epochs": 10,
   "save_steps": 500,
   "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 3,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 3
+      }
+    },
     "TrainerControl": {
       "args": {
         "should_epoch_stop": false,
@@ -175,7 +149,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 8.68297895581778e+16,
+  "total_flos": 1.2128346681348096e+16,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:633d6603fb5d519d493770c00d2039db41cce516bb4399ad40e6d01187a3b828
+oid sha256:7a0dba0afd6c07504c9951fbbc2017ec670cad8a3b2ca6951fe367b50a131172
 size 5624