Model save

README.md

@@ -1,6 +1,6 @@
 ---
 license: mit
-base_model: HuggingFaceH4/mistral-7b-sft-beta
+base_model: HuggingFaceH4/zephyr-7b-beta
 tags:
 - generated_from_trainer
 model-index:
@@ -13,17 +13,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-lora
 
-This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
+This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6704
-- Rewards/chosen: 0.0154
-- Rewards/rejected: -0.0291
-- Rewards/accuracies: 0.5898
-- Rewards/margins: 0.0445
-- Logps/rejected: -263.0990
-- Logps/chosen: -268.9026
-- Logits/rejected: -2.9178
-- Logits/chosen: -2.8992
+- Loss: 0.6874
+- Rewards/chosen: 0.0803
+- Rewards/rejected: 0.0298
+- Rewards/accuracies: 1.0
+- Rewards/margins: 0.0505
+- Logps/rejected: -101.0604
+- Logps/chosen: -102.9630
+- Logits/rejected: -2.2160
+- Logits/chosen: -2.1724
 
 ## Model description
 
@@ -54,15 +54,16 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 3
+- num_epochs: 5
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6836        | 0.99  | 120  | 0.6827          | 0.0088         | -0.0081          | 0.5742             | 0.0169          | -262.8899      | -268.9695    | -2.9192         | -2.9002       |
-| 0.6739        | 2.0   | 241  | 0.6739          | 0.0186         | -0.0224          | 0.6367             | 0.0410          | -263.0327      | -268.8711    | -2.9181         | -2.8995       |
-| 0.6692        | 2.98  | 360  | 0.6704          | 0.0154         | -0.0291          | 0.5898             | 0.0445          | -263.0990      | -268.9026    | -2.9178         | -2.8992       |
+| 0.6931        | 0.8   | 1    | 0.6931          | 0.0            | 0.0              | 0.0                | 0.0             | -101.3584      | -103.7664    | -2.2157         | -2.1717       |
+| 0.6931        | 1.6   | 2    | 0.6948          | 0.0296         | 0.0079           | 0.5                | 0.0217          | -101.2790      | -103.4700    | -2.2147         | -2.1715       |
+| 0.6931        | 2.4   | 3    | 0.6913          | 0.0277         | 0.0090           | 0.75               | 0.0188          | -101.2689      | -103.4891    | -2.2153         | -2.1709       |
+| 0.6931        | 4.0   | 5    | 0.6874          | 0.0803         | 0.0298           | 1.0                | 0.0505          | -101.0604      | -102.9630    | -2.2160         | -2.1724       |
 
 
 ### Framework versions

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "HuggingFaceH4/mistral-7b-sft-beta",
+  "base_model_name_or_path": "HuggingFaceH4/zephyr-7b-beta",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -16,10 +16,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "q_proj",
     "k_proj",
-    "o_proj"
+    "q_proj",
+    "o_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a470dc3d526d820ea9f9f2492aae51122fc16615cdd94c4e50f0ceffd25ce6f
+oid sha256:c85a9785f9eb434dc9c53abbc8850cebbf70813cbc5cdaaee5b484364b861dc6
 size 109086672

all_results.json

@@ -1,21 +1,21 @@
 {
-    "epoch": 2.98,
-    "eval_logits/chosen": -2.8992161750793457,
-    "eval_logits/rejected": -2.917783260345459,
-    "eval_logps/chosen": -268.90264892578125,
-    "eval_logps/rejected": -263.0989685058594,
-    "eval_loss": 0.6703784465789795,
-    "eval_rewards/accuracies": 0.58984375,
-    "eval_rewards/chosen": 0.015448413789272308,
-    "eval_rewards/margins": 0.04450809210538864,
-    "eval_rewards/rejected": -0.029059680178761482,
-    "eval_runtime": 181.4906,
-    "eval_samples": 2030,
-    "eval_samples_per_second": 11.185,
-    "eval_steps_per_second": 0.353,
-    "train_loss": 0.6791177903612454,
-    "train_runtime": 30079.4021,
-    "train_samples": 61761,
-    "train_samples_per_second": 6.16,
-    "train_steps_per_second": 0.012
+    "epoch": 4.0,
+    "eval_logits/chosen": -2.1724095344543457,
+    "eval_logits/rejected": -2.2160496711730957,
+    "eval_logps/chosen": -102.96302795410156,
+    "eval_logps/rejected": -101.06044006347656,
+    "eval_loss": 0.6873850226402283,
+    "eval_rewards/accuracies": 1.0,
+    "eval_rewards/chosen": 0.08033924549818039,
+    "eval_rewards/margins": 0.05054035410284996,
+    "eval_rewards/rejected": 0.02979888767004013,
+    "eval_runtime": 6.0716,
+    "eval_samples": 30,
+    "eval_samples_per_second": 4.941,
+    "eval_steps_per_second": 0.165,
+    "train_loss": 0.6922631859779358,
+    "train_runtime": 883.8621,
+    "train_samples": 626,
+    "train_samples_per_second": 3.541,
+    "train_steps_per_second": 0.006
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 2.98,
-    "eval_logits/chosen": -2.8992161750793457,
-    "eval_logits/rejected": -2.917783260345459,
-    "eval_logps/chosen": -268.90264892578125,
-    "eval_logps/rejected": -263.0989685058594,
-    "eval_loss": 0.6703784465789795,
-    "eval_rewards/accuracies": 0.58984375,
-    "eval_rewards/chosen": 0.015448413789272308,
-    "eval_rewards/margins": 0.04450809210538864,
-    "eval_rewards/rejected": -0.029059680178761482,
-    "eval_runtime": 181.4906,
-    "eval_samples": 2030,
-    "eval_samples_per_second": 11.185,
-    "eval_steps_per_second": 0.353
+    "epoch": 4.0,
+    "eval_logits/chosen": -2.1724095344543457,
+    "eval_logits/rejected": -2.2160496711730957,
+    "eval_logps/chosen": -102.96302795410156,
+    "eval_logps/rejected": -101.06044006347656,
+    "eval_loss": 0.6873850226402283,
+    "eval_rewards/accuracies": 1.0,
+    "eval_rewards/chosen": 0.08033924549818039,
+    "eval_rewards/margins": 0.05054035410284996,
+    "eval_rewards/rejected": 0.02979888767004013,
+    "eval_runtime": 6.0716,
+    "eval_samples": 30,
+    "eval_samples_per_second": 4.941,
+    "eval_steps_per_second": 0.165
 }

runs/Jan25_18-44-21_jupyter-dli/events.out.tfevents.1706208337.jupyter-dli.239378.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a19afef61062f8d1d2d156e1ea2b84f47e3b6dd69219235caa1e0dcd6503d50
+size 8264

runs/Jan25_18-44-21_jupyter-dli/events.out.tfevents.1706209227.jupyter-dli.239378.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49d267448f7319c8c27c184a48758ea3f229604df5725408467935f659db63d9
+size 815

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.98,
-    "train_loss": 0.6791177903612454,
-    "train_runtime": 30079.4021,
-    "train_samples": 61761,
-    "train_samples_per_second": 6.16,
-    "train_steps_per_second": 0.012
+    "epoch": 4.0,
+    "train_loss": 0.6922631859779358,
+    "train_runtime": 883.8621,
+    "train_samples": 626,
+    "train_samples_per_second": 3.541,
+    "train_steps_per_second": 0.006
 }

trainer_state.json

@@ -1,20 +1,20 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.983682983682984,
+  "epoch": 4.0,
   "eval_steps": 100,
-  "global_step": 360,
+  "global_step": 5,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.01,
-      "learning_rate": 1.3888888888888887e-08,
-      "logits/chosen": -2.951993465423584,
-      "logits/rejected": -2.9205567836761475,
-      "logps/chosen": -276.23077392578125,
-      "logps/rejected": -244.7322998046875,
+      "epoch": 0.8,
+      "learning_rate": 5e-07,
+      "logits/chosen": -2.3972699642181396,
+      "logits/rejected": -2.39332914352417,
+      "logps/chosen": -153.26783752441406,
+      "logps/rejected": -146.77935791015625,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -23,570 +23,82 @@
       "step": 1
     },
     {
-      "epoch": 0.08,
-      "learning_rate": 1.3888888888888888e-07,
-      "logits/chosen": -2.930194854736328,
-      "logits/rejected": -2.9322521686553955,
-      "logps/chosen": -258.4957275390625,
-      "logps/rejected": -252.5615234375,
-      "loss": 0.6937,
-      "rewards/accuracies": 0.4392361044883728,
-      "rewards/chosen": -0.0008609014330431819,
-      "rewards/margins": -0.0007996508502401412,
-      "rewards/rejected": -6.125055369921029e-05,
-      "step": 10
-    },
-    {
-      "epoch": 0.17,
-      "learning_rate": 2.7777777777777776e-07,
-      "logits/chosen": -2.935396432876587,
-      "logits/rejected": -2.9414877891540527,
-      "logps/chosen": -269.8578796386719,
-      "logps/rejected": -250.003662109375,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.0003611525462474674,
-      "rewards/margins": 0.002654502633959055,
-      "rewards/rejected": -0.0030156546272337437,
-      "step": 20
-    },
-    {
-      "epoch": 0.25,
-      "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.922468423843384,
-      "logits/rejected": -2.924722671508789,
-      "logps/chosen": -283.06756591796875,
-      "logps/rejected": -263.92694091796875,
-      "loss": 0.6926,
-      "rewards/accuracies": 0.503125011920929,
-      "rewards/chosen": 0.0008422272512689233,
-      "rewards/margins": 0.0005572644877247512,
-      "rewards/rejected": 0.00028496290906332433,
-      "step": 30
-    },
-    {
-      "epoch": 0.33,
-      "learning_rate": 4.938271604938271e-07,
-      "logits/chosen": -2.917189598083496,
-      "logits/rejected": -2.939058780670166,
-      "logps/chosen": -280.8387145996094,
-      "logps/rejected": -257.81353759765625,
-      "loss": 0.6915,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0037843205500394106,
-      "rewards/margins": 0.006630954332649708,
-      "rewards/rejected": -0.002846634481102228,
-      "step": 40
-    },
-    {
-      "epoch": 0.41,
-      "learning_rate": 4.783950617283951e-07,
-      "logits/chosen": -2.9359450340270996,
-      "logits/rejected": -2.93603777885437,
-      "logps/chosen": -268.5244445800781,
-      "logps/rejected": -253.9750518798828,
-      "loss": 0.6907,
-      "rewards/accuracies": 0.5453125238418579,
-      "rewards/chosen": 0.00316084292717278,
-      "rewards/margins": 0.005354513414204121,
-      "rewards/rejected": -0.0021936697885394096,
-      "step": 50
-    },
-    {
-      "epoch": 0.5,
-      "learning_rate": 4.6296296296296297e-07,
-      "logits/chosen": -2.942714214324951,
-      "logits/rejected": -2.9400081634521484,
-      "logps/chosen": -280.0882263183594,
-      "logps/rejected": -257.28839111328125,
-      "loss": 0.6892,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0040632085874676704,
-      "rewards/margins": 0.004900970496237278,
-      "rewards/rejected": -0.0008377617341466248,
-      "step": 60
-    },
-    {
-      "epoch": 0.58,
-      "learning_rate": 4.4753086419753083e-07,
-      "logits/chosen": -2.9372458457946777,
-      "logits/rejected": -2.9435629844665527,
-      "logps/chosen": -273.5423889160156,
-      "logps/rejected": -253.7714080810547,
-      "loss": 0.6882,
-      "rewards/accuracies": 0.565625011920929,
-      "rewards/chosen": 0.0066502392292022705,
-      "rewards/margins": 0.010684076696634293,
-      "rewards/rejected": -0.004033837933093309,
-      "step": 70
-    },
-    {
-      "epoch": 0.66,
-      "learning_rate": 4.320987654320987e-07,
-      "logits/chosen": -2.896692991256714,
-      "logits/rejected": -2.875420093536377,
-      "logps/chosen": -274.2783508300781,
-      "logps/rejected": -255.4978790283203,
-      "loss": 0.6883,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 0.007802584674209356,
-      "rewards/margins": 0.010292423889040947,
-      "rewards/rejected": -0.0024898392148315907,
-      "step": 80
-    },
-    {
-      "epoch": 0.75,
-      "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.923689603805542,
-      "logits/rejected": -2.935713291168213,
-      "logps/chosen": -277.1148986816406,
-      "logps/rejected": -247.6132354736328,
-      "loss": 0.6867,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 0.007110255304723978,
-      "rewards/margins": 0.012844784185290337,
-      "rewards/rejected": -0.005734528414905071,
-      "step": 90
-    },
-    {
-      "epoch": 0.83,
-      "learning_rate": 4.0123456790123453e-07,
-      "logits/chosen": -2.9391238689422607,
-      "logits/rejected": -2.931495189666748,
-      "logps/chosen": -273.63140869140625,
-      "logps/rejected": -253.19534301757812,
-      "loss": 0.6857,
-      "rewards/accuracies": 0.5703125,
-      "rewards/chosen": 0.0076993731781840324,
-      "rewards/margins": 0.014409579336643219,
-      "rewards/rejected": -0.0067102061584591866,
-      "step": 100
-    },
-    {
-      "epoch": 0.91,
-      "learning_rate": 3.8580246913580245e-07,
-      "logits/chosen": -2.9282491207122803,
-      "logits/rejected": -2.924396276473999,
-      "logps/chosen": -264.43011474609375,
-      "logps/rejected": -243.5092315673828,
-      "loss": 0.6851,
-      "rewards/accuracies": 0.5796874761581421,
-      "rewards/chosen": 0.007344271056354046,
-      "rewards/margins": 0.016184944659471512,
-      "rewards/rejected": -0.008840671740472317,
-      "step": 110
-    },
-    {
-      "epoch": 0.99,
-      "learning_rate": 3.703703703703703e-07,
-      "logits/chosen": -2.9296412467956543,
-      "logits/rejected": -2.9385550022125244,
-      "logps/chosen": -280.14300537109375,
-      "logps/rejected": -244.4756317138672,
-      "loss": 0.6836,
-      "rewards/accuracies": 0.6234375238418579,
-      "rewards/chosen": 0.011619888246059418,
-      "rewards/margins": 0.023034537211060524,
-      "rewards/rejected": -0.011414647102355957,
-      "step": 120
-    },
-    {
-      "epoch": 0.99,
-      "eval_logits/chosen": -2.900207042694092,
-      "eval_logits/rejected": -2.9191529750823975,
-      "eval_logps/chosen": -268.9694519042969,
-      "eval_logps/rejected": -262.8898620605469,
-      "eval_loss": 0.6826778650283813,
-      "eval_rewards/accuracies": 0.57421875,
-      "eval_rewards/chosen": 0.008766621351242065,
-      "eval_rewards/margins": 0.016915924847126007,
-      "eval_rewards/rejected": -0.008149303495883942,
-      "eval_runtime": 182.2275,
-      "eval_samples_per_second": 11.14,
-      "eval_steps_per_second": 0.351,
-      "step": 120
-    },
-    {
-      "epoch": 1.08,
-      "learning_rate": 3.549382716049383e-07,
-      "logits/chosen": -2.9227781295776367,
-      "logits/rejected": -2.901576519012451,
-      "logps/chosen": -259.58624267578125,
-      "logps/rejected": -248.39675903320312,
-      "loss": 0.6816,
-      "rewards/accuracies": 0.6156250238418579,
-      "rewards/chosen": 0.010982013307511806,
-      "rewards/margins": 0.02251209318637848,
-      "rewards/rejected": -0.011530080810189247,
-      "step": 130
-    },
-    {
-      "epoch": 1.16,
-      "learning_rate": 3.3950617283950614e-07,
-      "logits/chosen": -2.9140427112579346,
-      "logits/rejected": -2.903498649597168,
-      "logps/chosen": -267.62469482421875,
-      "logps/rejected": -242.287109375,
-      "loss": 0.6816,
-      "rewards/accuracies": 0.598437488079071,
-      "rewards/chosen": 0.013705916702747345,
-      "rewards/margins": 0.021453356370329857,
-      "rewards/rejected": -0.007747439201921225,
-      "step": 140
-    },
-    {
-      "epoch": 1.24,
-      "learning_rate": 3.2407407407407406e-07,
-      "logits/chosen": -2.928597927093506,
-      "logits/rejected": -2.9360718727111816,
-      "logps/chosen": -283.0503845214844,
-      "logps/rejected": -263.2129821777344,
-      "loss": 0.6812,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.016042985022068024,
-      "rewards/margins": 0.02658357098698616,
-      "rewards/rejected": -0.010540584102272987,
-      "step": 150
-    },
-    {
-      "epoch": 1.33,
-      "learning_rate": 3.086419753086419e-07,
-      "logits/chosen": -2.9245991706848145,
-      "logits/rejected": -2.9161458015441895,
-      "logps/chosen": -283.2376403808594,
-      "logps/rejected": -249.5203399658203,
-      "loss": 0.6786,
-      "rewards/accuracies": 0.620312511920929,
-      "rewards/chosen": 0.017403725534677505,
-      "rewards/margins": 0.030968856066465378,
-      "rewards/rejected": -0.013565132394433022,
-      "step": 160
-    },
-    {
-      "epoch": 1.41,
-      "learning_rate": 2.932098765432099e-07,
-      "logits/chosen": -2.9159512519836426,
-      "logits/rejected": -2.9223380088806152,
-      "logps/chosen": -272.6524353027344,
-      "logps/rejected": -249.50711059570312,
-      "loss": 0.6799,
-      "rewards/accuracies": 0.6234375238418579,
-      "rewards/chosen": 0.01738204061985016,
-      "rewards/margins": 0.02959570661187172,
-      "rewards/rejected": -0.01221366599202156,
-      "step": 170
-    },
-    {
-      "epoch": 1.49,
-      "learning_rate": 2.7777777777777776e-07,
-      "logits/chosen": -2.9411699771881104,
-      "logits/rejected": -2.936310291290283,
-      "logps/chosen": -259.71112060546875,
-      "logps/rejected": -245.5165252685547,
-      "loss": 0.6786,
-      "rewards/accuracies": 0.609375,
-      "rewards/chosen": 0.01514382939785719,
-      "rewards/margins": 0.02933506667613983,
-      "rewards/rejected": -0.014191237278282642,
-      "step": 180
-    },
-    {
-      "epoch": 1.57,
-      "learning_rate": 2.623456790123457e-07,
-      "logits/chosen": -2.932964563369751,
-      "logits/rejected": -2.9182915687561035,
-      "logps/chosen": -267.070556640625,
-      "logps/rejected": -242.4732208251953,
-      "loss": 0.6763,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.010820230469107628,
-      "rewards/margins": 0.028901537880301476,
-      "rewards/rejected": -0.018081307411193848,
-      "step": 190
-    },
-    {
-      "epoch": 1.66,
-      "learning_rate": 2.4691358024691354e-07,
-      "logits/chosen": -2.9549834728240967,
-      "logits/rejected": -2.9499382972717285,
-      "logps/chosen": -274.26885986328125,
-      "logps/rejected": -251.1107940673828,
-      "loss": 0.6763,
-      "rewards/accuracies": 0.6234375238418579,
-      "rewards/chosen": 0.016665898263454437,
-      "rewards/margins": 0.03335059434175491,
-      "rewards/rejected": -0.016684692353010178,
-      "step": 200
-    },
-    {
-      "epoch": 1.74,
-      "learning_rate": 2.3148148148148148e-07,
-      "logits/chosen": -2.922349214553833,
-      "logits/rejected": -2.9319396018981934,
-      "logps/chosen": -277.7991027832031,
-      "logps/rejected": -265.2469482421875,
-      "loss": 0.6756,
-      "rewards/accuracies": 0.620312511920929,
-      "rewards/chosen": 0.017026837915182114,
-      "rewards/margins": 0.03320584446191788,
-      "rewards/rejected": -0.016179006546735764,
-      "step": 210
-    },
-    {
-      "epoch": 1.82,
-      "learning_rate": 2.1604938271604935e-07,
-      "logits/chosen": -2.921238899230957,
-      "logits/rejected": -2.91098952293396,
-      "logps/chosen": -265.0746154785156,
-      "logps/rejected": -245.3384246826172,
-      "loss": 0.6745,
-      "rewards/accuracies": 0.660937488079071,
-      "rewards/chosen": 0.01762019656598568,
-      "rewards/margins": 0.04026446118950844,
-      "rewards/rejected": -0.02264426089823246,
-      "step": 220
-    },
-    {
-      "epoch": 1.91,
-      "learning_rate": 2.0061728395061726e-07,
-      "logits/chosen": -2.957491636276245,
-      "logits/rejected": -2.9489035606384277,
-      "logps/chosen": -269.3310546875,
-      "logps/rejected": -258.45452880859375,
-      "loss": 0.6727,
-      "rewards/accuracies": 0.6484375,
-      "rewards/chosen": 0.02087206393480301,
-      "rewards/margins": 0.04010782390832901,
-      "rewards/rejected": -0.01923576183617115,
-      "step": 230
-    },
-    {
-      "epoch": 1.99,
-      "learning_rate": 1.8518518518518516e-07,
-      "logits/chosen": -2.912625789642334,
-      "logits/rejected": -2.8954434394836426,
-      "logps/chosen": -278.33441162109375,
-      "logps/rejected": -257.1600341796875,
-      "loss": 0.6739,
-      "rewards/accuracies": 0.6484375,
-      "rewards/chosen": 0.019938599318265915,
-      "rewards/margins": 0.043915756046772,
-      "rewards/rejected": -0.023977158591151237,
-      "step": 240
-    },
-    {
-      "epoch": 2.0,
-      "eval_logits/chosen": -2.8995361328125,
-      "eval_logits/rejected": -2.918098211288452,
-      "eval_logps/chosen": -268.8711242675781,
-      "eval_logps/rejected": -263.03265380859375,
-      "eval_loss": 0.6739373207092285,
-      "eval_rewards/accuracies": 0.63671875,
-      "eval_rewards/chosen": 0.01860005408525467,
-      "eval_rewards/margins": 0.04102998971939087,
-      "eval_rewards/rejected": -0.02242993377149105,
-      "eval_runtime": 181.6795,
-      "eval_samples_per_second": 11.174,
-      "eval_steps_per_second": 0.352,
-      "step": 241
-    },
-    {
-      "epoch": 2.07,
-      "learning_rate": 1.6975308641975307e-07,
-      "logits/chosen": -2.9307079315185547,
-      "logits/rejected": -2.936490535736084,
-      "logps/chosen": -279.3055725097656,
-      "logps/rejected": -257.22015380859375,
-      "loss": 0.6722,
-      "rewards/accuracies": 0.6031249761581421,
-      "rewards/chosen": 0.01767881028354168,
-      "rewards/margins": 0.041353292763233185,
-      "rewards/rejected": -0.023674478754401207,
-      "step": 250
-    },
-    {
-      "epoch": 2.15,
-      "learning_rate": 1.5432098765432096e-07,
-      "logits/chosen": -2.879582405090332,
-      "logits/rejected": -2.8889400959014893,
-      "logps/chosen": -257.8949279785156,
-      "logps/rejected": -241.9752197265625,
-      "loss": 0.6723,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.019200006499886513,
-      "rewards/margins": 0.0419248566031456,
-      "rewards/rejected": -0.022724846377968788,
-      "step": 260
-    },
-    {
-      "epoch": 2.24,
-      "learning_rate": 1.3888888888888888e-07,
-      "logits/chosen": -2.9449806213378906,
-      "logits/rejected": -2.932685136795044,
-      "logps/chosen": -267.568359375,
-      "logps/rejected": -242.255859375,
-      "loss": 0.6739,
-      "rewards/accuracies": 0.676562488079071,
-      "rewards/chosen": 0.023365100845694542,
-      "rewards/margins": 0.046661119908094406,
-      "rewards/rejected": -0.023296022787690163,
-      "step": 270
+      "epoch": 0.8,
+      "eval_logits/chosen": -2.1716835498809814,
+      "eval_logits/rejected": -2.2157046794891357,
+      "eval_logps/chosen": -103.76641845703125,
+      "eval_logps/rejected": -101.35842895507812,
+      "eval_loss": 0.6931473016738892,
+      "eval_rewards/accuracies": 0.0,
+      "eval_rewards/chosen": 0.0,
+      "eval_rewards/margins": 0.0,
+      "eval_rewards/rejected": 0.0,
+      "eval_runtime": 6.5465,
+      "eval_samples_per_second": 4.583,
+      "eval_steps_per_second": 0.153,
+      "step": 1
     },
     {
-      "epoch": 2.32,
-      "learning_rate": 1.2345679012345677e-07,
-      "logits/chosen": -2.9022796154022217,
-      "logits/rejected": -2.9109320640563965,
-      "logps/chosen": -265.966064453125,
-      "logps/rejected": -239.6620330810547,
-      "loss": 0.6707,
-      "rewards/accuracies": 0.676562488079071,
-      "rewards/chosen": 0.026586908847093582,
-      "rewards/margins": 0.05420393496751785,
-      "rewards/rejected": -0.02761702612042427,
-      "step": 280
+      "epoch": 1.6,
+      "eval_logits/chosen": -2.1715452671051025,
+      "eval_logits/rejected": -2.2146592140197754,
+      "eval_logps/chosen": -103.4699935913086,
+      "eval_logps/rejected": -101.27898406982422,
+      "eval_loss": 0.694814920425415,
+      "eval_rewards/accuracies": 0.5,
+      "eval_rewards/chosen": 0.02964324876666069,
+      "eval_rewards/margins": 0.021698763594031334,
+      "eval_rewards/rejected": 0.007944487035274506,
+      "eval_runtime": 6.0506,
+      "eval_samples_per_second": 4.958,
+      "eval_steps_per_second": 0.165,
+      "step": 2
     },
     {
       "epoch": 2.4,
-      "learning_rate": 1.0802469135802467e-07,
-      "logits/chosen": -2.9430556297302246,
-      "logits/rejected": -2.946108341217041,
-      "logps/chosen": -278.1526794433594,
-      "logps/rejected": -263.77178955078125,
-      "loss": 0.6717,
-      "rewards/accuracies": 0.6328125,
-      "rewards/chosen": 0.019661400467157364,
-      "rewards/margins": 0.047070957720279694,
-      "rewards/rejected": -0.02740955725312233,
-      "step": 290
-    },
-    {
-      "epoch": 2.49,
-      "learning_rate": 9.259259259259258e-08,
-      "logits/chosen": -2.9402213096618652,
-      "logits/rejected": -2.9220597743988037,
-      "logps/chosen": -260.75665283203125,
-      "logps/rejected": -263.7254943847656,
-      "loss": 0.6701,
-      "rewards/accuracies": 0.667187511920929,
-      "rewards/chosen": 0.017683709040284157,
-      "rewards/margins": 0.04577410966157913,
-      "rewards/rejected": -0.028090402483940125,
-      "step": 300
-    },
-    {
-      "epoch": 2.57,
-      "learning_rate": 7.716049382716048e-08,
-      "logits/chosen": -2.9286868572235107,
-      "logits/rejected": -2.9299063682556152,
-      "logps/chosen": -264.2181701660156,
-      "logps/rejected": -261.91363525390625,
-      "loss": 0.6704,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.01738332025706768,
-      "rewards/margins": 0.0443243645131588,
-      "rewards/rejected": -0.02694104239344597,
-      "step": 310
-    },
-    {
-      "epoch": 2.65,
-      "learning_rate": 6.172839506172839e-08,
-      "logits/chosen": -2.9626357555389404,
-      "logits/rejected": -2.951683521270752,
-      "logps/chosen": -283.7442626953125,
-      "logps/rejected": -256.7106018066406,
-      "loss": 0.6709,
-      "rewards/accuracies": 0.6546875238418579,
-      "rewards/chosen": 0.024839671328663826,
-      "rewards/margins": 0.05458993837237358,
-      "rewards/rejected": -0.029750261455774307,
-      "step": 320
-    },
-    {
-      "epoch": 2.74,
-      "learning_rate": 4.629629629629629e-08,
-      "logits/chosen": -2.9203758239746094,
-      "logits/rejected": -2.898723840713501,
-      "logps/chosen": -275.98834228515625,
-      "logps/rejected": -259.5394592285156,
-      "loss": 0.67,
-      "rewards/accuracies": 0.6265624761581421,
-      "rewards/chosen": 0.021389978006482124,
-      "rewards/margins": 0.0422237329185009,
-      "rewards/rejected": -0.020833751186728477,
-      "step": 330
-    },
-    {
-      "epoch": 2.82,
-      "learning_rate": 3.086419753086419e-08,
-      "logits/chosen": -2.9191930294036865,
-      "logits/rejected": -2.918905735015869,
-      "logps/chosen": -282.91143798828125,
-      "logps/rejected": -272.44012451171875,
-      "loss": 0.6683,
-      "rewards/accuracies": 0.640625,
-      "rewards/chosen": 0.026901666074991226,
-      "rewards/margins": 0.055442653596401215,
-      "rewards/rejected": -0.028540989384055138,
-      "step": 340
-    },
-    {
-      "epoch": 2.9,
-      "learning_rate": 1.5432098765432096e-08,
-      "logits/chosen": -2.9336206912994385,
-      "logits/rejected": -2.929389715194702,
-      "logps/chosen": -292.12884521484375,
-      "logps/rejected": -262.97174072265625,
-      "loss": 0.6696,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.03554671257734299,
-      "rewards/margins": 0.07022932916879654,
-      "rewards/rejected": -0.034682609140872955,
-      "step": 350
-    },
-    {
-      "epoch": 2.98,
-      "learning_rate": 0.0,
-      "logits/chosen": -2.918612003326416,
-      "logits/rejected": -2.92228627204895,
-      "logps/chosen": -272.35125732421875,
-      "logps/rejected": -258.17169189453125,
-      "loss": 0.6692,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.030398529022932053,
-      "rewards/margins": 0.05524751543998718,
-      "rewards/rejected": -0.02484898641705513,
-      "step": 360
-    },
-    {
-      "epoch": 2.98,
-      "eval_logits/chosen": -2.8992161750793457,
-      "eval_logits/rejected": -2.917783260345459,
-      "eval_logps/chosen": -268.90264892578125,
-      "eval_logps/rejected": -263.0989685058594,
-      "eval_loss": 0.6703784465789795,
-      "eval_rewards/accuracies": 0.58984375,
-      "eval_rewards/chosen": 0.015448413789272308,
-      "eval_rewards/margins": 0.04450809210538864,
-      "eval_rewards/rejected": -0.029059680178761482,
-      "eval_runtime": 181.8484,
-      "eval_samples_per_second": 11.163,
-      "eval_steps_per_second": 0.352,
-      "step": 360
-    },
-    {
-      "epoch": 2.98,
-      "step": 360,
+      "eval_logits/chosen": -2.170850992202759,
+      "eval_logits/rejected": -2.215348482131958,
+      "eval_logps/chosen": -103.48912811279297,
+      "eval_logps/rejected": -101.26887512207031,
+      "eval_loss": 0.6913403868675232,
+      "eval_rewards/accuracies": 0.75,
+      "eval_rewards/chosen": 0.027730178087949753,
+      "eval_rewards/margins": 0.018774602562189102,
+      "eval_rewards/rejected": 0.00895557552576065,
+      "eval_runtime": 6.0481,
+      "eval_samples_per_second": 4.96,
+      "eval_steps_per_second": 0.165,
+      "step": 3
+    },
+    {
+      "epoch": 4.0,
+      "eval_logits/chosen": -2.1724095344543457,
+      "eval_logits/rejected": -2.2160496711730957,
+      "eval_logps/chosen": -102.96302795410156,
+      "eval_logps/rejected": -101.06044006347656,
+      "eval_loss": 0.6873850226402283,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.08033924549818039,
+      "eval_rewards/margins": 0.05054035410284996,
+      "eval_rewards/rejected": 0.02979888767004013,
+      "eval_runtime": 6.0751,
+      "eval_samples_per_second": 4.938,
+      "eval_steps_per_second": 0.165,
+      "step": 5
+    },
+    {
+      "epoch": 4.0,
+      "step": 5,
       "total_flos": 0.0,
-      "train_loss": 0.6791177903612454,
-      "train_runtime": 30079.4021,
-      "train_samples_per_second": 6.16,
-      "train_steps_per_second": 0.012
+      "train_loss": 0.6922631859779358,
+      "train_runtime": 883.8621,
+      "train_samples_per_second": 3.541,
+      "train_steps_per_second": 0.006
     }
   ],
   "logging_steps": 10,
-  "max_steps": 360,
-  "num_train_epochs": 3,
+  "max_steps": 5,
+  "num_train_epochs": 5,
   "save_steps": 500,
   "total_flos": 0.0,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c8227384e82904ba8d02ed7dd3cf55663930c5840142dfed7da06bbbc5bda5e
+oid sha256:b1679b26c1e5cc6b5a159fba09aee53f44a4515086460ae0d7e21b31df550a03
 size 5752