phi-3.5-new

adapter_config.json

@@ -23,9 +23,9 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
     "qkv_proj",
     "o_proj",
+    "down_proj",
     "gate_up_proj"
   ],
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41b60cffda93a140952b4e8e313fd8a8519ca094dd39b7008a23a9942e80caa3
+oid sha256:b1b8b56f3b700a6bea1e2bca037d2506f4811f2d8270799c66980c14fe48dc69
 size 100697728

all_results.json

@@ -1,12 +1,12 @@
 {
-    "epoch": 7.0,
-    "eval_loss": 1.0098934173583984,
-    "eval_runtime": 7.0732,
-    "eval_samples_per_second": 2.403,
-    "eval_steps_per_second": 1.272,
-    "total_flos": 1.2128346681348096e+16,
-    "train_loss": 0.8996787428044949,
-    "train_runtime": 846.9703,
-    "train_samples_per_second": 0.98,
-    "train_steps_per_second": 0.496
+    "epoch": 11.0,
+    "eval_loss": 0.7707116603851318,
+    "eval_runtime": 103.4112,
+    "eval_samples_per_second": 2.476,
+    "eval_steps_per_second": 1.238,
+    "total_flos": 2.8711375226284032e+17,
+    "train_loss": 0.7217305723603789,
+    "train_runtime": 19548.1756,
+    "train_samples_per_second": 1.596,
+    "train_steps_per_second": 0.798
 }

eval_results.json

@@ -1,7 +1,7 @@
 {
-    "epoch": 7.0,
-    "eval_loss": 1.0098934173583984,
-    "eval_runtime": 7.0732,
-    "eval_samples_per_second": 2.403,
-    "eval_steps_per_second": 1.272
+    "epoch": 11.0,
+    "eval_loss": 0.7707116603851318,
+    "eval_runtime": 103.4112,
+    "eval_samples_per_second": 2.476,
+    "eval_steps_per_second": 1.238
 }

runs/Jan12_19-05-30_dmlab/events.out.tfevents.1736701530.dmlab.9923.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea451e6836705c4213d569c92050dce2d40dd61fefa3e76493bf24617e46571f
+size 8942

runs/Jan12_19-26-23_dmlab/events.out.tfevents.1736702784.dmlab.12960.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c3af953317924b74894174a19e6891e5da60c2ee9a62af28194155428ca9fd6
+size 13979

runs/Jan12_19-26-23_dmlab/events.out.tfevents.1736722436.dmlab.12960.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6258b6e6405a2a28ee922fa51f7132627d4c1bebe946d9260579999a61bcffe7
+size 359

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 7.0,
-    "total_flos": 1.2128346681348096e+16,
-    "train_loss": 0.8996787428044949,
-    "train_runtime": 846.9703,
-    "train_samples_per_second": 0.98,
-    "train_steps_per_second": 0.496
+    "epoch": 11.0,
+    "total_flos": 2.8711375226284032e+17,
+    "train_loss": 0.7217305723603789,
+    "train_runtime": 19548.1756,
+    "train_samples_per_second": 1.596,
+    "train_steps_per_second": 0.798
 }

trainer_state.json

@@ -1,132 +1,192 @@
 {
-  "best_metric": 1.0029878616333008,
-  "best_model_checkpoint": "/home/labuser/Documents/phi-3/phi-3.5-new/checkpoint-168",
-  "epoch": 7.0,
+  "best_metric": 0.7596490383148193,
+  "best_model_checkpoint": "/home/labuser/Documents/phi-3/phi-3.5-new/checkpoint-4992",
+  "epoch": 11.0,
   "eval_steps": 500,
-  "global_step": 294,
+  "global_step": 6864,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 1.0,
-      "grad_norm": 1.0487970113754272,
-      "learning_rate": 0.0001,
-      "loss": 1.1905,
-      "step": 42
+      "grad_norm": 1.0626534223556519,
+      "learning_rate": 4e-05,
+      "loss": 0.9984,
+      "step": 624
     },
     {
       "epoch": 1.0,
-      "eval_loss": 1.0386497974395752,
-      "eval_runtime": 7.0656,
-      "eval_samples_per_second": 2.406,
-      "eval_steps_per_second": 1.274,
-      "step": 42
+      "eval_loss": 0.969778299331665,
+      "eval_runtime": 103.4178,
+      "eval_samples_per_second": 2.475,
+      "eval_steps_per_second": 1.238,
+      "step": 624
     },
     {
       "epoch": 2.0,
-      "grad_norm": 1.0445537567138672,
-      "learning_rate": 0.0002,
-      "loss": 0.9483,
-      "step": 84
+      "grad_norm": 0.967179000377655,
+      "learning_rate": 8e-05,
+      "loss": 0.8711,
+      "step": 1248
     },
     {
       "epoch": 2.0,
-      "eval_loss": 1.0180531740188599,
-      "eval_runtime": 7.0935,
-      "eval_samples_per_second": 2.397,
-      "eval_steps_per_second": 1.269,
-      "step": 84
+      "eval_loss": 0.9493655562400818,
+      "eval_runtime": 103.4116,
+      "eval_samples_per_second": 2.476,
+      "eval_steps_per_second": 1.238,
+      "step": 1248
     },
     {
       "epoch": 3.0,
-      "grad_norm": 0.8360362648963928,
-      "learning_rate": 0.0001923879532511287,
-      "loss": 0.9012,
-      "step": 126
+      "grad_norm": 1.005494475364685,
+      "learning_rate": 0.00012,
+      "loss": 0.837,
+      "step": 1872
     },
     {
       "epoch": 3.0,
-      "eval_loss": 1.0081464052200317,
-      "eval_runtime": 7.1072,
-      "eval_samples_per_second": 2.392,
-      "eval_steps_per_second": 1.266,
-      "step": 126
+      "eval_loss": 0.9314318895339966,
+      "eval_runtime": 103.4002,
+      "eval_samples_per_second": 2.476,
+      "eval_steps_per_second": 1.238,
+      "step": 1872
     },
     {
       "epoch": 4.0,
-      "grad_norm": 0.9580802917480469,
-      "learning_rate": 0.00017071067811865476,
-      "loss": 0.8626,
-      "step": 168
+      "grad_norm": 0.9488267302513123,
+      "learning_rate": 0.00016,
+      "loss": 0.8053,
+      "step": 2496
     },
     {
       "epoch": 4.0,
-      "eval_loss": 1.0029878616333008,
-      "eval_runtime": 7.081,
-      "eval_samples_per_second": 2.401,
-      "eval_steps_per_second": 1.271,
-      "step": 168
+      "eval_loss": 0.9051965475082397,
+      "eval_runtime": 103.4268,
+      "eval_samples_per_second": 2.475,
+      "eval_steps_per_second": 1.238,
+      "step": 2496
     },
     {
       "epoch": 5.0,
-      "grad_norm": 0.8999230861663818,
-      "learning_rate": 0.000138268343236509,
-      "loss": 0.8324,
-      "step": 210
+      "grad_norm": 1.2727611064910889,
+      "learning_rate": 0.0002,
+      "loss": 0.7577,
+      "step": 3120
     },
     {
       "epoch": 5.0,
-      "eval_loss": 1.0067561864852905,
-      "eval_runtime": 7.0886,
-      "eval_samples_per_second": 2.398,
-      "eval_steps_per_second": 1.27,
-      "step": 210
+      "eval_loss": 0.8548531532287598,
+      "eval_runtime": 103.3949,
+      "eval_samples_per_second": 2.476,
+      "eval_steps_per_second": 1.238,
+      "step": 3120
     },
     {
       "epoch": 6.0,
-      "grad_norm": 0.8453378081321716,
-      "learning_rate": 0.0001,
-      "loss": 0.8004,
-      "step": 252
+      "grad_norm": 1.3708724975585938,
+      "learning_rate": 0.00019876883405951377,
+      "loss": 0.6823,
+      "step": 3744
     },
     {
       "epoch": 6.0,
-      "eval_loss": 1.0179213285446167,
-      "eval_runtime": 7.0766,
-      "eval_samples_per_second": 2.402,
-      "eval_steps_per_second": 1.272,
-      "step": 252
+      "eval_loss": 0.7896583080291748,
+      "eval_runtime": 103.416,
+      "eval_samples_per_second": 2.475,
+      "eval_steps_per_second": 1.238,
+      "step": 3744
     },
     {
       "epoch": 7.0,
-      "grad_norm": 1.0025222301483154,
-      "learning_rate": 6.173165676349103e-05,
-      "loss": 0.7622,
-      "step": 294
+      "grad_norm": 1.065096378326416,
+      "learning_rate": 0.00019510565162951537,
+      "loss": 0.6376,
+      "step": 4368
     },
     {
       "epoch": 7.0,
-      "eval_loss": 1.043684959411621,
-      "eval_runtime": 7.0812,
-      "eval_samples_per_second": 2.401,
-      "eval_steps_per_second": 1.271,
-      "step": 294
+      "eval_loss": 0.7802127003669739,
+      "eval_runtime": 103.439,
+      "eval_samples_per_second": 2.475,
+      "eval_steps_per_second": 1.237,
+      "step": 4368
     },
     {
-      "epoch": 7.0,
-      "step": 294,
-      "total_flos": 1.2128346681348096e+16,
-      "train_loss": 0.8996787428044949,
-      "train_runtime": 846.9703,
-      "train_samples_per_second": 0.98,
-      "train_steps_per_second": 0.496
+      "epoch": 8.0,
+      "grad_norm": 0.973934531211853,
+      "learning_rate": 0.0001891006524188368,
+      "loss": 0.6122,
+      "step": 4992
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 0.7596490383148193,
+      "eval_runtime": 103.3881,
+      "eval_samples_per_second": 2.476,
+      "eval_steps_per_second": 1.238,
+      "step": 4992
+    },
+    {
+      "epoch": 9.0,
+      "grad_norm": 1.0052335262298584,
+      "learning_rate": 0.00018090169943749476,
+      "loss": 0.5947,
+      "step": 5616
+    },
+    {
+      "epoch": 9.0,
+      "eval_loss": 0.7636829614639282,
+      "eval_runtime": 103.3982,
+      "eval_samples_per_second": 2.476,
+      "eval_steps_per_second": 1.238,
+      "step": 5616
+    },
+    {
+      "epoch": 10.0,
+      "grad_norm": 0.9973880648612976,
+      "learning_rate": 0.00017071067811865476,
+      "loss": 0.5797,
+      "step": 6240
+    },
+    {
+      "epoch": 10.0,
+      "eval_loss": 0.7768124341964722,
+      "eval_runtime": 103.4116,
+      "eval_samples_per_second": 2.476,
+      "eval_steps_per_second": 1.238,
+      "step": 6240
+    },
+    {
+      "epoch": 11.0,
+      "grad_norm": 0.9684802293777466,
+      "learning_rate": 0.00015877852522924732,
+      "loss": 0.5631,
+      "step": 6864
+    },
+    {
+      "epoch": 11.0,
+      "eval_loss": 0.7780652046203613,
+      "eval_runtime": 103.4007,
+      "eval_samples_per_second": 2.476,
+      "eval_steps_per_second": 1.238,
+      "step": 6864
+    },
+    {
+      "epoch": 11.0,
+      "step": 6864,
+      "total_flos": 2.8711375226284032e+17,
+      "train_loss": 0.7217305723603789,
+      "train_runtime": 19548.1756,
+      "train_samples_per_second": 1.596,
+      "train_steps_per_second": 0.798
     }
   ],
   "logging_steps": 500,
-  "max_steps": 420,
+  "max_steps": 15600,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 10,
+  "num_train_epochs": 25,
   "save_steps": 500,
   "stateful_callbacks": {
     "EarlyStoppingCallback": {
@@ -149,7 +209,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.2128346681348096e+16,
+  "total_flos": 2.8711375226284032e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a0dba0afd6c07504c9951fbbc2017ec670cad8a3b2ca6951fe367b50a131172
+oid sha256:e29c43b50ee6b41567386050e54799b545a4237e251ef4c7b3fefda200b7073d
 size 5624