phi-3.5-new

adapter_config.json

@@ -23,10 +23,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "qkv_proj",
     "o_proj",
-    "down_proj",
-    "gate_up_proj"
+    "qkv_proj",
+    "gate_up_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1b8b56f3b700a6bea1e2bca037d2506f4811f2d8270799c66980c14fe48dc69
+oid sha256:272a00f063966eb6a188503f16dfaafeeb0b3938fb064be3e153d64dfa16cb4b
 size 100697728

all_results.json

@@ -1,12 +1,12 @@
 {
-    "epoch": 11.0,
-    "eval_loss": 0.7707116603851318,
-    "eval_runtime": 103.4112,
-    "eval_samples_per_second": 2.476,
-    "eval_steps_per_second": 1.238,
-    "total_flos": 2.8711375226284032e+17,
-    "train_loss": 0.7217305723603789,
-    "train_runtime": 19548.1756,
-    "train_samples_per_second": 1.596,
-    "train_steps_per_second": 0.798
+    "epoch": 5.0,
+    "eval_loss": 1.006455421447754,
+    "eval_runtime": 6.8379,
+    "eval_samples_per_second": 2.486,
+    "eval_steps_per_second": 0.731,
+    "total_flos": 8689108767160320.0,
+    "train_loss": 0.9555120922270275,
+    "train_runtime": 582.4007,
+    "train_samples_per_second": 0.713,
+    "train_steps_per_second": 0.18
 }

eval_results.json

@@ -1,7 +1,7 @@
 {
-    "epoch": 11.0,
-    "eval_loss": 0.7707116603851318,
-    "eval_runtime": 103.4112,
-    "eval_samples_per_second": 2.476,
-    "eval_steps_per_second": 1.238
+    "epoch": 5.0,
+    "eval_loss": 1.006455421447754,
+    "eval_runtime": 6.8379,
+    "eval_samples_per_second": 2.486,
+    "eval_steps_per_second": 0.731
 }

runs/Jan13_05-28-33_dmlab/events.out.tfevents.1736738913.dmlab.67479.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f803481d3f0bbfc18400647f693e840fdc101cf29ac62a44aabcefa2ec9c61c
+size 8794

runs/Jan13_05-38-09_dmlab/events.out.tfevents.1736739489.dmlab.68165.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffd48dd236376b12f91682cd6a2fb7eb986c728357ceab6c97f976c4f755ee01
+size 9263

runs/Jan13_05-43-08_dmlab/events.out.tfevents.1736739789.dmlab.69294.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:805dc2394501b132f8f71f3ea9f2fb04fc7f9f2a510fa132e43b670de0f2589e
+size 11030

runs/Jan13_05-43-08_dmlab/events.out.tfevents.1736740378.dmlab.69294.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9000e42f8dca2f9562ef60c3b7766ecae263c5aa2fab3ba365006576109a279
+size 354

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 11.0,
-    "total_flos": 2.8711375226284032e+17,
-    "train_loss": 0.7217305723603789,
-    "train_runtime": 19548.1756,
-    "train_samples_per_second": 1.596,
-    "train_steps_per_second": 0.798
+    "epoch": 5.0,
+    "total_flos": 8689108767160320.0,
+    "train_loss": 0.9555120922270275,
+    "train_runtime": 582.4007,
+    "train_samples_per_second": 0.713,
+    "train_steps_per_second": 0.18
 }

trainer_state.json

@@ -1,192 +1,102 @@
 {
-  "best_metric": 0.7596490383148193,
-  "best_model_checkpoint": "/home/labuser/Documents/phi-3/phi-3.5-new/checkpoint-4992",
-  "epoch": 11.0,
+  "best_metric": 0.9966219663619995,
+  "best_model_checkpoint": "/home/labuser/Documents/phi-3/phi-3.5-new/checkpoint-63",
+  "epoch": 5.0,
   "eval_steps": 500,
-  "global_step": 6864,
+  "global_step": 105,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 1.0,
-      "grad_norm": 1.0626534223556519,
-      "learning_rate": 4e-05,
-      "loss": 0.9984,
-      "step": 624
+      "grad_norm": 0.5774533748626709,
+      "learning_rate": 0.0002,
+      "loss": 1.2076,
+      "step": 21
     },
     {
       "epoch": 1.0,
-      "eval_loss": 0.969778299331665,
-      "eval_runtime": 103.4178,
-      "eval_samples_per_second": 2.475,
-      "eval_steps_per_second": 1.238,
-      "step": 624
+      "eval_loss": 1.060381531715393,
+      "eval_runtime": 6.8834,
+      "eval_samples_per_second": 2.47,
+      "eval_steps_per_second": 0.726,
+      "step": 21
     },
     {
       "epoch": 2.0,
-      "grad_norm": 0.967179000377655,
-      "learning_rate": 8e-05,
-      "loss": 0.8711,
-      "step": 1248
+      "grad_norm": 0.4814026951789856,
+      "learning_rate": 0.00017071067811865476,
+      "loss": 0.9571,
+      "step": 42
     },
     {
       "epoch": 2.0,
-      "eval_loss": 0.9493655562400818,
-      "eval_runtime": 103.4116,
-      "eval_samples_per_second": 2.476,
-      "eval_steps_per_second": 1.238,
-      "step": 1248
+      "eval_loss": 0.998464047908783,
+      "eval_runtime": 6.8587,
+      "eval_samples_per_second": 2.479,
+      "eval_steps_per_second": 0.729,
+      "step": 42
     },
     {
       "epoch": 3.0,
-      "grad_norm": 1.005494475364685,
-      "learning_rate": 0.00012,
-      "loss": 0.837,
-      "step": 1872
+      "grad_norm": 0.5674614906311035,
+      "learning_rate": 0.0001,
+      "loss": 0.901,
+      "step": 63
     },
     {
       "epoch": 3.0,
-      "eval_loss": 0.9314318895339966,
-      "eval_runtime": 103.4002,
+      "eval_loss": 0.9966219663619995,
+      "eval_runtime": 6.8664,
       "eval_samples_per_second": 2.476,
-      "eval_steps_per_second": 1.238,
-      "step": 1872
+      "eval_steps_per_second": 0.728,
+      "step": 63
     },
     {
       "epoch": 4.0,
-      "grad_norm": 0.9488267302513123,
-      "learning_rate": 0.00016,
-      "loss": 0.8053,
-      "step": 2496
+      "grad_norm": 0.5171260833740234,
+      "learning_rate": 2.9289321881345254e-05,
+      "loss": 0.8659,
+      "step": 84
     },
     {
       "epoch": 4.0,
-      "eval_loss": 0.9051965475082397,
-      "eval_runtime": 103.4268,
-      "eval_samples_per_second": 2.475,
-      "eval_steps_per_second": 1.238,
-      "step": 2496
+      "eval_loss": 1.001924753189087,
+      "eval_runtime": 6.8571,
+      "eval_samples_per_second": 2.479,
+      "eval_steps_per_second": 0.729,
+      "step": 84
     },
     {
       "epoch": 5.0,
-      "grad_norm": 1.2727611064910889,
-      "learning_rate": 0.0002,
-      "loss": 0.7577,
-      "step": 3120
+      "grad_norm": 0.4767768681049347,
+      "learning_rate": 0.0,
+      "loss": 0.846,
+      "step": 105
     },
     {
       "epoch": 5.0,
-      "eval_loss": 0.8548531532287598,
-      "eval_runtime": 103.3949,
-      "eval_samples_per_second": 2.476,
-      "eval_steps_per_second": 1.238,
-      "step": 3120
-    },
-    {
-      "epoch": 6.0,
-      "grad_norm": 1.3708724975585938,
-      "learning_rate": 0.00019876883405951377,
-      "loss": 0.6823,
-      "step": 3744
-    },
-    {
-      "epoch": 6.0,
-      "eval_loss": 0.7896583080291748,
-      "eval_runtime": 103.416,
-      "eval_samples_per_second": 2.475,
-      "eval_steps_per_second": 1.238,
-      "step": 3744
-    },
-    {
-      "epoch": 7.0,
-      "grad_norm": 1.065096378326416,
-      "learning_rate": 0.00019510565162951537,
-      "loss": 0.6376,
-      "step": 4368
-    },
-    {
-      "epoch": 7.0,
-      "eval_loss": 0.7802127003669739,
-      "eval_runtime": 103.439,
-      "eval_samples_per_second": 2.475,
-      "eval_steps_per_second": 1.237,
-      "step": 4368
-    },
-    {
-      "epoch": 8.0,
-      "grad_norm": 0.973934531211853,
-      "learning_rate": 0.0001891006524188368,
-      "loss": 0.6122,
-      "step": 4992
-    },
-    {
-      "epoch": 8.0,
-      "eval_loss": 0.7596490383148193,
-      "eval_runtime": 103.3881,
-      "eval_samples_per_second": 2.476,
-      "eval_steps_per_second": 1.238,
-      "step": 4992
+      "eval_loss": 1.0021475553512573,
+      "eval_runtime": 6.8536,
+      "eval_samples_per_second": 2.48,
+      "eval_steps_per_second": 0.73,
+      "step": 105
     },
     {
-      "epoch": 9.0,
-      "grad_norm": 1.0052335262298584,
-      "learning_rate": 0.00018090169943749476,
-      "loss": 0.5947,
-      "step": 5616
-    },
-    {
-      "epoch": 9.0,
-      "eval_loss": 0.7636829614639282,
-      "eval_runtime": 103.3982,
-      "eval_samples_per_second": 2.476,
-      "eval_steps_per_second": 1.238,
-      "step": 5616
-    },
-    {
-      "epoch": 10.0,
-      "grad_norm": 0.9973880648612976,
-      "learning_rate": 0.00017071067811865476,
-      "loss": 0.5797,
-      "step": 6240
-    },
-    {
-      "epoch": 10.0,
-      "eval_loss": 0.7768124341964722,
-      "eval_runtime": 103.4116,
-      "eval_samples_per_second": 2.476,
-      "eval_steps_per_second": 1.238,
-      "step": 6240
-    },
-    {
-      "epoch": 11.0,
-      "grad_norm": 0.9684802293777466,
-      "learning_rate": 0.00015877852522924732,
-      "loss": 0.5631,
-      "step": 6864
-    },
-    {
-      "epoch": 11.0,
-      "eval_loss": 0.7780652046203613,
-      "eval_runtime": 103.4007,
-      "eval_samples_per_second": 2.476,
-      "eval_steps_per_second": 1.238,
-      "step": 6864
-    },
-    {
-      "epoch": 11.0,
-      "step": 6864,
-      "total_flos": 2.8711375226284032e+17,
-      "train_loss": 0.7217305723603789,
-      "train_runtime": 19548.1756,
-      "train_samples_per_second": 1.596,
-      "train_steps_per_second": 0.798
+      "epoch": 5.0,
+      "step": 105,
+      "total_flos": 8689108767160320.0,
+      "train_loss": 0.9555120922270275,
+      "train_runtime": 582.4007,
+      "train_samples_per_second": 0.713,
+      "train_steps_per_second": 0.18
     }
   ],
   "logging_steps": 500,
-  "max_steps": 15600,
+  "max_steps": 105,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 25,
+  "num_train_epochs": 5,
   "save_steps": 500,
   "stateful_callbacks": {
     "EarlyStoppingCallback": {
@@ -195,7 +105,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 3
+        "early_stopping_patience_counter": 2
       }
     },
     "TrainerControl": {
@@ -209,8 +119,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.8711375226284032e+17,
-  "train_batch_size": 2,
+  "total_flos": 8689108767160320.0,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e29c43b50ee6b41567386050e54799b545a4237e251ef4c7b3fefda200b7073d
+oid sha256:b13726b299e8575d2863caea80e5c8126effe8fd59991a7918fa2d8a2e87a4b9
 size 5624