Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6941
-- Rewards/chosen: -0.0008
-- Rewards/rejected: 0.0011
-- Rewards/accuracies: 0.4870
-- Rewards/margins: -0.0019
-- Logps/rejected: -256.6144
-- Logps/chosen: -273.1258
-- Logits/rejected: -2.8923
-- Logits/chosen: -2.8913
+- Loss: 0.6925
+- Rewards/chosen: -0.0045
+- Rewards/rejected: 0.0164
+- Rewards/accuracies: 0.25
+- Rewards/margins: -0.0209
+- Logps/rejected: -35.6209
+- Logps/chosen: -119.3746
+- Logits/rejected: -2.6567
+- Logits/chosen: -2.7227
 
 ## Model description
 
@@ -47,10 +47,10 @@ The following hyperparameters were used during training:
 - eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 2
+- num_devices: 8
 - gradient_accumulation_steps: 32
-- total_train_batch_size: 128
-- total_eval_batch_size: 8
+- total_train_batch_size: 512
+- total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.1
@@ -60,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6931        | 0.84  | 4    | 0.6944          | -0.0020        | 0.0023           | 0.4680             | -0.0042         | -256.6025      | -273.1375    | -2.8922         | -2.8913       |
-| 0.6931        | 1.88  | 9    | 0.6937          | -0.0006        | 0.0011           | 0.4930             | -0.0016         | -256.6148      | -273.1236    | -2.8922         | -2.8914       |
-| 0.6934        | 2.51  | 12   | 0.6941          | -0.0008        | 0.0011           | 0.4870             | -0.0019         | -256.6144      | -273.1258    | -2.8923         | -2.8913       |
+| 0.6931        | 0.8   | 1    | 0.6931          | 0.0            | 0.0              | 0.0                | 0.0             | -35.7851       | -119.3297    | -2.6563         | -2.7224       |
+| 0.6931        | 1.6   | 2    | 0.6903          | -0.0086        | 0.0118           | 0.0                | -0.0204         | -35.6673       | -119.4162    | -2.6570         | -2.7221       |
+| 0.6931        | 2.4   | 3    | 0.6925          | -0.0045        | 0.0164           | 0.25               | -0.0209         | -35.6209       | -119.3746    | -2.6567         | -2.7227       |
 
 
 ### Framework versions

adapter_config.json

@@ -16,10 +16,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "k_proj",
     "q_proj",
-    "o_proj"
+    "v_proj",
+    "o_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9aa4b56ad055076743bb6f6bf61d2cfad2be482bbca7691c5ffb99fcd147dca
+oid sha256:98a53cb969618e88f0b672364127ce1906c69e16656f648c1eaccf6e536d36ca
 size 109086672

all_results.json

@@ -1,21 +1,21 @@
 {
-    "epoch": 2.51,
-    "eval_logits/chosen": -2.891324996948242,
-    "eval_logits/rejected": -2.892286777496338,
-    "eval_logps/chosen": -273.12579345703125,
-    "eval_logps/rejected": -256.6144104003906,
-    "eval_loss": 0.6940562725067139,
-    "eval_rewards/accuracies": 0.4869999885559082,
-    "eval_rewards/chosen": -0.0007941981311887503,
-    "eval_rewards/margins": -0.0018857381073758006,
-    "eval_rewards/rejected": 0.0010915396269410849,
-    "eval_runtime": 605.1943,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 3.305,
-    "eval_steps_per_second": 0.413,
-    "train_loss": 0.6932857781648636,
-    "train_runtime": 2636.7323,
-    "train_samples": 611,
-    "train_samples_per_second": 0.695,
-    "train_steps_per_second": 0.005
+    "epoch": 2.4,
+    "eval_logits/chosen": -2.7227089405059814,
+    "eval_logits/rejected": -2.6567294597625732,
+    "eval_logps/chosen": -119.37458038330078,
+    "eval_logps/rejected": -35.62090301513672,
+    "eval_loss": 0.6925258040428162,
+    "eval_rewards/accuracies": 0.25,
+    "eval_rewards/chosen": -0.004486369900405407,
+    "eval_rewards/margins": -0.020908452570438385,
+    "eval_rewards/rejected": 0.016422081738710403,
+    "eval_runtime": 2.6911,
+    "eval_samples": 30,
+    "eval_samples_per_second": 11.148,
+    "eval_steps_per_second": 0.372,
+    "train_loss": 0.6927651365598043,
+    "train_runtime": 245.4034,
+    "train_samples": 626,
+    "train_samples_per_second": 7.653,
+    "train_steps_per_second": 0.012
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 2.51,
-    "eval_logits/chosen": -2.891324996948242,
-    "eval_logits/rejected": -2.892286777496338,
-    "eval_logps/chosen": -273.12579345703125,
-    "eval_logps/rejected": -256.6144104003906,
-    "eval_loss": 0.6940562725067139,
-    "eval_rewards/accuracies": 0.4869999885559082,
-    "eval_rewards/chosen": -0.0007941981311887503,
-    "eval_rewards/margins": -0.0018857381073758006,
-    "eval_rewards/rejected": 0.0010915396269410849,
-    "eval_runtime": 605.1943,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 3.305,
-    "eval_steps_per_second": 0.413
+    "epoch": 2.4,
+    "eval_logits/chosen": -2.7227089405059814,
+    "eval_logits/rejected": -2.6567294597625732,
+    "eval_logps/chosen": -119.37458038330078,
+    "eval_logps/rejected": -35.62090301513672,
+    "eval_loss": 0.6925258040428162,
+    "eval_rewards/accuracies": 0.25,
+    "eval_rewards/chosen": -0.004486369900405407,
+    "eval_rewards/margins": -0.020908452570438385,
+    "eval_rewards/rejected": 0.016422081738710403,
+    "eval_runtime": 2.6911,
+    "eval_samples": 30,
+    "eval_samples_per_second": 11.148,
+    "eval_steps_per_second": 0.372
 }

runs/Jan25_03-04-25_jupyter-dli/events.out.tfevents.1706151962.jupyter-dli.54020.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:342c7b14c459551375d416fb868b77b7a0941e1d8c22d3c24eacc792f56b1067
+size 4991

runs/Jan25_03-26-17_jupyter-dli/events.out.tfevents.1706153211.jupyter-dli.63500.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ba9daeacc69a50eb4f81cc2618ecfed56e9e9e0516febce196499eefb7e8e49
+size 7520

runs/Jan25_03-26-17_jupyter-dli/events.out.tfevents.1706153459.jupyter-dli.63500.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1352996015b9bdb9b7f09f5d0228224b5f6d7426df19a9faf500a9ea4b4665ec
+size 815

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.51,
-    "train_loss": 0.6932857781648636,
-    "train_runtime": 2636.7323,
-    "train_samples": 611,
-    "train_samples_per_second": 0.695,
-    "train_steps_per_second": 0.005
+    "epoch": 2.4,
+    "train_loss": 0.6927651365598043,
+    "train_runtime": 245.4034,
+    "train_samples": 626,
+    "train_samples_per_second": 7.653,
+    "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -1,20 +1,20 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.5098039215686274,
+  "epoch": 2.4,
   "eval_steps": 100,
-  "global_step": 12,
+  "global_step": 3,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.21,
-      "learning_rate": 2.5e-07,
-      "logits/chosen": -2.9307734966278076,
-      "logits/rejected": -2.9072206020355225,
-      "logps/chosen": -257.9442138671875,
-      "logps/rejected": -223.16062927246094,
+      "epoch": 0.8,
+      "learning_rate": 5e-07,
+      "logits/chosen": -2.7074732780456543,
+      "logits/rejected": -2.714259147644043,
+      "logps/chosen": -177.91046142578125,
+      "logps/rejected": -169.544921875,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -23,79 +23,65 @@
       "step": 1
     },
     {
-      "epoch": 0.84,
-      "eval_logits/chosen": -2.8913073539733887,
-      "eval_logits/rejected": -2.892223834991455,
-      "eval_logps/chosen": -273.1374816894531,
-      "eval_logps/rejected": -256.60247802734375,
-      "eval_loss": 0.6944335699081421,
-      "eval_rewards/accuracies": 0.46799999475479126,
-      "eval_rewards/chosen": -0.0019581823144108057,
-      "eval_rewards/margins": -0.004248426295816898,
-      "eval_rewards/rejected": 0.0022902432829141617,
-      "eval_runtime": 604.0376,
-      "eval_samples_per_second": 3.311,
-      "eval_steps_per_second": 0.414,
-      "step": 4
-    },
-    {
-      "epoch": 1.88,
-      "eval_logits/chosen": -2.891376495361328,
-      "eval_logits/rejected": -2.8922488689422607,
-      "eval_logps/chosen": -273.12359619140625,
-      "eval_logps/rejected": -256.61480712890625,
-      "eval_loss": 0.6936609148979187,
-      "eval_rewards/accuracies": 0.49300000071525574,
-      "eval_rewards/chosen": -0.0005687248194590211,
-      "eval_rewards/margins": -0.0016233286587521434,
-      "eval_rewards/rejected": 0.0010546041885390878,
-      "eval_runtime": 607.6521,
-      "eval_samples_per_second": 3.291,
-      "eval_steps_per_second": 0.411,
-      "step": 9
+      "epoch": 0.8,
+      "eval_logits/chosen": -2.7223572731018066,
+      "eval_logits/rejected": -2.6562907695770264,
+      "eval_logps/chosen": -119.3297119140625,
+      "eval_logps/rejected": -35.78512191772461,
+      "eval_loss": 0.6931473016738892,
+      "eval_rewards/accuracies": 0.0,
+      "eval_rewards/chosen": 0.0,
+      "eval_rewards/margins": 0.0,
+      "eval_rewards/rejected": 0.0,
+      "eval_runtime": 2.722,
+      "eval_samples_per_second": 11.021,
+      "eval_steps_per_second": 0.367,
+      "step": 1
     },
     {
-      "epoch": 2.09,
-      "learning_rate": 1e-07,
-      "logits/chosen": -2.9432342052459717,
-      "logits/rejected": -2.949681520462036,
-      "logps/chosen": -269.6990051269531,
-      "logps/rejected": -258.2781982421875,
-      "loss": 0.6934,
-      "rewards/accuracies": 0.4340277910232544,
-      "rewards/chosen": 0.0009870771318674088,
-      "rewards/margins": 0.0012862730072811246,
-      "rewards/rejected": -0.0002991959627252072,
-      "step": 10
+      "epoch": 1.6,
+      "eval_logits/chosen": -2.722149133682251,
+      "eval_logits/rejected": -2.656991481781006,
+      "eval_logps/chosen": -119.41619873046875,
+      "eval_logps/rejected": -35.667266845703125,
+      "eval_loss": 0.690349817276001,
+      "eval_rewards/accuracies": 0.0,
+      "eval_rewards/chosen": -0.008648109622299671,
+      "eval_rewards/margins": -0.020433522760868073,
+      "eval_rewards/rejected": 0.011785412207245827,
+      "eval_runtime": 2.7089,
+      "eval_samples_per_second": 11.075,
+      "eval_steps_per_second": 0.369,
+      "step": 2
     },
     {
-      "epoch": 2.51,
-      "eval_logits/chosen": -2.891324996948242,
-      "eval_logits/rejected": -2.892286777496338,
-      "eval_logps/chosen": -273.12579345703125,
-      "eval_logps/rejected": -256.6144104003906,
-      "eval_loss": 0.6940562725067139,
-      "eval_rewards/accuracies": 0.4869999885559082,
-      "eval_rewards/chosen": -0.0007941981311887503,
-      "eval_rewards/margins": -0.0018857381073758006,
-      "eval_rewards/rejected": 0.0010915396269410849,
-      "eval_runtime": 604.8757,
-      "eval_samples_per_second": 3.306,
-      "eval_steps_per_second": 0.413,
-      "step": 12
+      "epoch": 2.4,
+      "eval_logits/chosen": -2.7227089405059814,
+      "eval_logits/rejected": -2.6567294597625732,
+      "eval_logps/chosen": -119.37458038330078,
+      "eval_logps/rejected": -35.62090301513672,
+      "eval_loss": 0.6925258040428162,
+      "eval_rewards/accuracies": 0.25,
+      "eval_rewards/chosen": -0.004486369900405407,
+      "eval_rewards/margins": -0.020908452570438385,
+      "eval_rewards/rejected": 0.016422081738710403,
+      "eval_runtime": 2.6892,
+      "eval_samples_per_second": 11.156,
+      "eval_steps_per_second": 0.372,
+      "step": 3
     },
     {
-      "epoch": 2.51,
-      "step": 12,
+      "epoch": 2.4,
+      "step": 3,
       "total_flos": 0.0,
-      "train_loss": 0.6932857781648636,
-      "train_runtime": 2636.7323,
-      "train_samples_per_second": 0.695,
-      "train_steps_per_second": 0.005
+      "train_loss": 0.6927651365598043,
+      "train_runtime": 245.4034,
+      "train_samples_per_second": 7.653,
+      "train_steps_per_second": 0.012
     }
   ],
   "logging_steps": 10,
-  "max_steps": 12,
+  "max_steps": 3,
   "num_train_epochs": 3,
   "save_steps": 500,
   "total_flos": 0.0,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6e0a992426eb70b08db763df2e71fa42f4afe271e674e50c228443e1dc5bb93
+oid sha256:f53e20161db1b1d41d7c2044c14f2ecec03643d3dd24b7f13d44742d874a8247
 size 5752