Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NorLLM-AI/NorMistral-7B
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: norllm-ai-normistral-7b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # norllm-ai-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/norllm-ai-normistral-7b-sft-qlora](https://huggingface.co/data/norllm-ai-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NorLLM-AI/NorMistral-7B](https://huggingface.co/NorLLM-AI/NorMistral-7B) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 6.0745
-- Rewards/chosen: -0.0067
-- Rewards/rejected: -0.0282
-- Rewards/accuracies: 0.5660
-- Rewards/margins: 0.0215
-- Logps/rejected: -34.8390
-- Logps/chosen: -31.3159
-- Logits/rejected: -2.8030
-- Logits/chosen: -2.8055
+- Loss: 0.9012
+- Rewards/chosen: -0.0642
+- Rewards/rejected: -0.1770
+- Rewards/accuracies: 0.5656
+- Rewards/margins: 0.1128
+- Logps/rejected: -34.9511
+- Logps/chosen: -31.3742
+- Logits/rejected: -2.8184
+- Logits/chosen: -2.8208
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 5.3752        | 0.26  | 100  | 6.0492          | 0.0006         | -0.0242          | 0.5893             | 0.0248          | -34.8191       | -31.2794     | -2.8110         | -2.8137       |
-| 5.6214        | 0.52  | 200  | 5.9543          | -0.0060        | -0.0319          | 0.5685             | 0.0259          | -34.8574       | -31.3123     | -2.8103         | -2.8130       |
-| 4.7461        | 0.78  | 300  | 5.9715          | 0.0004         | -0.0246          | 0.5748             | 0.0250          | -34.8210       | -31.2804     | -2.8025         | -2.8048       |
+| 0.8179        | 0.26  | 100  | 0.8995          | -0.0056        | -0.1202          | 0.6100             | 0.1146          | -34.8698       | -31.2905     | -2.8112         | -2.8139       |
+| 0.7257        | 0.52  | 200  | 0.9028          | -0.0688        | -0.1792          | 0.5573             | 0.1104          | -34.9542       | -31.3808     | -2.8244         | -2.8270       |
+| 0.6308        | 0.78  | 300  | 0.9012          | -0.0642        | -0.1770          | 0.5656             | 0.1128          | -34.9511       | -31.3742     | -2.8184         | -2.8208       |
 
 
 ### Framework versions

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "k_proj",
     "v_proj",
-    "gate_proj",
-    "down_proj",
     "o_proj",
-    "k_proj",
     "q_proj",
-    "up_proj"
+    "up_proj",
+    "down_proj",
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4090508b20c0e9de7ef5ea2fbbb7808e34eaea38c106d5c4811031af16a57f0f
+oid sha256:b281ed005b74d9d74523c7f202168f8d954a50afc1f0154f3f7d4101ab111404
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.805530071258545,
-    "eval_logits/rejected": -2.802974224090576,
-    "eval_logps/chosen": -31.315858840942383,
-    "eval_logps/rejected": -34.838993072509766,
-    "eval_loss": 6.074533939361572,
-    "eval_rewards/accuracies": 0.5660299062728882,
-    "eval_rewards/chosen": -0.006680913269519806,
-    "eval_rewards/margins": 0.021486198529601097,
-    "eval_rewards/rejected": -0.0281671155244112,
-    "eval_runtime": 112.8101,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.041,
-    "eval_steps_per_second": 0.381,
-    "train_loss": 4.931960544338474,
-    "train_runtime": 2724.0851,
+    "train_loss": 0.6684874992866021,
+    "train_runtime": 2718.5428,
     "train_samples": 3079,
-    "train_samples_per_second": 1.13,
-    "train_steps_per_second": 0.141
+    "train_samples_per_second": 1.133,
+    "train_steps_per_second": 0.142
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 4.931960544338474,
-    "train_runtime": 2724.0851,
+    "train_loss": 0.6684874992866021,
+    "train_runtime": 2718.5428,
     "train_samples": 3079,
-    "train_samples_per_second": 1.13,
-    "train_steps_per_second": 0.141
+    "train_samples_per_second": 1.133,
+    "train_steps_per_second": 0.142
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 396.0,
+      "grad_norm": 55.25,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": -2.7358343601226807,
       "logits/rejected": -2.7480404376983643,
       "logps/chosen": -27.35565757751465,
       "logps/rejected": -21.06114387512207,
-      "loss": 6.25,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,630 +25,630 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 672.0,
+      "grad_norm": 94.0,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -3.0095510482788086,
-      "logits/rejected": -2.9984350204467773,
-      "logps/chosen": -33.197166442871094,
-      "logps/rejected": -31.98773193359375,
-      "loss": 6.2548,
-      "rewards/accuracies": 0.4861111044883728,
-      "rewards/chosen": -0.0032450438011437654,
-      "rewards/margins": 0.0015198996989056468,
-      "rewards/rejected": -0.004764944314956665,
+      "logits/chosen": -3.009542942047119,
+      "logits/rejected": -2.998302698135376,
+      "logps/chosen": -33.17807388305664,
+      "logps/rejected": -31.984642028808594,
+      "loss": 0.9835,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.002009392250329256,
+      "rewards/margins": 0.016525447368621826,
+      "rewards/rejected": -0.014516051858663559,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 480.0,
+      "grad_norm": 64.5,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -2.8992972373962402,
-      "logits/rejected": -2.8939919471740723,
-      "logps/chosen": -32.48074722290039,
-      "logps/rejected": -28.958911895751953,
-      "loss": 6.4479,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.0011298510944470763,
-      "rewards/margins": -0.005721274763345718,
-      "rewards/rejected": 0.004591422621160746,
+      "logits/chosen": -2.899103879928589,
+      "logits/rejected": -2.8936753273010254,
+      "logps/chosen": -32.4957160949707,
+      "logps/rejected": -28.984477996826172,
+      "loss": 1.0126,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.014431963674724102,
+      "rewards/margins": -0.012606342323124409,
+      "rewards/rejected": -0.0018256217008456588,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 426.0,
+      "grad_norm": 62.0,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -3.097346067428589,
-      "logits/rejected": -3.1091365814208984,
-      "logps/chosen": -32.795387268066406,
-      "logps/rejected": -30.13690185546875,
-      "loss": 6.087,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.02266053855419159,
-      "rewards/margins": 0.01083013229072094,
-      "rewards/rejected": 0.011830407194793224,
+      "logits/chosen": -3.0970749855041504,
+      "logits/rejected": -3.1094558238983154,
+      "logps/chosen": -32.78588104248047,
+      "logps/rejected": -30.1519775390625,
+      "loss": 0.9449,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.08596263825893402,
+      "rewards/margins": 0.05510985851287842,
+      "rewards/rejected": 0.030852774158120155,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 376.0,
+      "grad_norm": 50.5,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.8621602058410645,
-      "logits/rejected": -2.852827310562134,
-      "logps/chosen": -31.5306396484375,
-      "logps/rejected": -32.392234802246094,
-      "loss": 5.4681,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.05993650108575821,
-      "rewards/margins": 0.06689226627349854,
-      "rewards/rejected": -0.006955766584724188,
+      "logits/chosen": -2.862823486328125,
+      "logits/rejected": -2.853950023651123,
+      "logps/chosen": -31.538238525390625,
+      "logps/rejected": -32.37445068359375,
+      "loss": 0.8574,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.20445938408374786,
+      "rewards/margins": 0.21635785698890686,
+      "rewards/rejected": -0.011898472905158997,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 330.0,
+      "grad_norm": 39.0,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -2.881068468093872,
-      "logits/rejected": -2.8789007663726807,
-      "logps/chosen": -29.444555282592773,
-      "logps/rejected": -30.105321884155273,
-      "loss": 5.8178,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.05912144109606743,
-      "rewards/margins": 0.06239712983369827,
-      "rewards/rejected": -0.003275694791227579,
+      "logits/chosen": -2.881490707397461,
+      "logits/rejected": -2.8795459270477295,
+      "logps/chosen": -29.380319595336914,
+      "logps/rejected": -30.1256046295166,
+      "loss": 0.8579,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.2518887519836426,
+      "rewards/margins": 0.2775513231754303,
+      "rewards/rejected": -0.025662561878561974,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 378.0,
+      "grad_norm": 55.75,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -2.9106826782226562,
-      "logits/rejected": -2.9123446941375732,
-      "logps/chosen": -29.89569091796875,
-      "logps/rejected": -28.091384887695312,
-      "loss": 4.9965,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.046815626323223114,
-      "rewards/margins": 0.07395622134208679,
-      "rewards/rejected": -0.027140596881508827,
+      "logits/chosen": -2.9107794761657715,
+      "logits/rejected": -2.912201166152954,
+      "logps/chosen": -29.885101318359375,
+      "logps/rejected": -28.084728240966797,
+      "loss": 0.804,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.17126679420471191,
+      "rewards/margins": 0.26159828901290894,
+      "rewards/rejected": -0.0903315395116806,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 1256.0,
+      "grad_norm": 72.0,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.9927854537963867,
-      "logits/rejected": -2.9993896484375,
-      "logps/chosen": -29.258081436157227,
-      "logps/rejected": -31.0142822265625,
-      "loss": 6.6005,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.004577371757477522,
-      "rewards/margins": 0.029073869809508324,
-      "rewards/rejected": -0.024496497586369514,
+      "logits/chosen": -2.9932219982147217,
+      "logits/rejected": -2.999392032623291,
+      "logps/chosen": -29.26175308227539,
+      "logps/rejected": -31.045358657836914,
+      "loss": 0.8981,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.013451325707137585,
+      "rewards/margins": 0.12094201892614365,
+      "rewards/rejected": -0.10749070346355438,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 552.0,
+      "grad_norm": 61.75,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -2.8086979389190674,
-      "logits/rejected": -2.823762893676758,
-      "logps/chosen": -29.380992889404297,
-      "logps/rejected": -29.88627052307129,
-      "loss": 5.3018,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.031947556883096695,
-      "rewards/margins": 0.07434491813182831,
-      "rewards/rejected": -0.04239736869931221,
+      "logits/chosen": -2.808314561843872,
+      "logits/rejected": -2.8239831924438477,
+      "logps/chosen": -29.382043838500977,
+      "logps/rejected": -29.915264129638672,
+      "loss": 0.8143,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.11108281463384628,
+      "rewards/margins": 0.27977070212364197,
+      "rewards/rejected": -0.1686878651380539,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 424.0,
+      "grad_norm": 48.75,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -2.8993382453918457,
-      "logits/rejected": -2.8812999725341797,
-      "logps/chosen": -32.65332794189453,
-      "logps/rejected": -30.278905868530273,
-      "loss": 9.3008,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.03662032634019852,
-      "rewards/margins": 0.08807238936424255,
-      "rewards/rejected": -0.05145206302404404,
+      "logits/chosen": -2.898421049118042,
+      "logits/rejected": -2.8805272579193115,
+      "logps/chosen": -32.720703125,
+      "logps/rejected": -30.371923446655273,
+      "loss": 0.8446,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.08101295679807663,
+      "rewards/margins": 0.3262065052986145,
+      "rewards/rejected": -0.2451934814453125,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 332.0,
+      "grad_norm": 48.25,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -3.0058953762054443,
-      "logits/rejected": -3.0061850547790527,
-      "logps/chosen": -31.760330200195312,
-      "logps/rejected": -30.88376808166504,
-      "loss": 5.3752,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.05930906534194946,
-      "rewards/margins": 0.07282786071300507,
-      "rewards/rejected": -0.013518787920475006,
+      "logits/chosen": -3.005709409713745,
+      "logits/rejected": -3.0060715675354004,
+      "logps/chosen": -31.795307159423828,
+      "logps/rejected": -30.9265193939209,
+      "loss": 0.8179,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.18309839069843292,
+      "rewards/margins": 0.26034125685691833,
+      "rewards/rejected": -0.07724283635616302,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.8137402534484863,
-      "eval_logits/rejected": -2.810950994491577,
-      "eval_logps/chosen": -31.279376983642578,
-      "eval_logps/rejected": -34.8190803527832,
-      "eval_loss": 6.049151420593262,
-      "eval_rewards/accuracies": 0.5892857313156128,
-      "eval_rewards/chosen": 0.0006145605584606528,
-      "eval_rewards/margins": 0.024798991158604622,
-      "eval_rewards/rejected": -0.02418443188071251,
-      "eval_runtime": 113.3821,
+      "eval_logits/chosen": -2.813856840133667,
+      "eval_logits/rejected": -2.8111777305603027,
+      "eval_logps/chosen": -31.290462493896484,
+      "eval_logps/rejected": -34.86981201171875,
+      "eval_loss": 0.8994626402854919,
+      "eval_rewards/accuracies": 0.610049843788147,
+      "eval_rewards/chosen": -0.005606824532151222,
+      "eval_rewards/margins": 0.11455333232879639,
+      "eval_rewards/rejected": -0.12016014754772186,
+      "eval_runtime": 113.4039,
       "eval_samples_per_second": 3.025,
       "eval_steps_per_second": 0.379,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 524.0,
+      "grad_norm": 73.0,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -2.961501359939575,
-      "logits/rejected": -2.9370064735412598,
-      "logps/chosen": -31.725719451904297,
-      "logps/rejected": -31.284133911132812,
-      "loss": 5.3812,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.09821529686450958,
-      "rewards/margins": 0.12327277660369873,
-      "rewards/rejected": -0.02505749836564064,
+      "logits/chosen": -2.9637725353240967,
+      "logits/rejected": -2.9400086402893066,
+      "logps/chosen": -31.802236557006836,
+      "logps/rejected": -31.42850685119629,
+      "loss": 0.7127,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.29019027948379517,
+      "rewards/margins": 0.47895127534866333,
+      "rewards/rejected": -0.18876102566719055,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 380.0,
+      "grad_norm": 57.25,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -3.0424726009368896,
-      "logits/rejected": -3.0713469982147217,
-      "logps/chosen": -28.6945858001709,
-      "logps/rejected": -34.26778793334961,
-      "loss": 5.044,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.09386318176984787,
-      "rewards/margins": 0.11791019141674042,
-      "rewards/rejected": -0.024047017097473145,
+      "logits/chosen": -3.0474579334259033,
+      "logits/rejected": -3.076024055480957,
+      "logps/chosen": -28.733871459960938,
+      "logps/rejected": -34.316707611083984,
+      "loss": 0.7493,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.3010229170322418,
+      "rewards/margins": 0.41943326592445374,
+      "rewards/rejected": -0.11841034889221191,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 294.0,
+      "grad_norm": 46.25,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.7436156272888184,
-      "logits/rejected": -2.7392959594726562,
-      "logps/chosen": -28.56683349609375,
-      "logps/rejected": -30.19887924194336,
-      "loss": 5.5435,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.08156983554363251,
-      "rewards/margins": 0.0945839211344719,
-      "rewards/rejected": -0.01301408838480711,
+      "logits/chosen": -2.7504048347473145,
+      "logits/rejected": -2.7461349964141846,
+      "logps/chosen": -28.568979263305664,
+      "logps/rejected": -30.313528060913086,
+      "loss": 0.7475,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.28399184346199036,
+      "rewards/margins": 0.4097979664802551,
+      "rewards/rejected": -0.12580609321594238,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 238.0,
+      "grad_norm": 40.75,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -3.0176939964294434,
-      "logits/rejected": -3.01535964012146,
-      "logps/chosen": -27.101856231689453,
-      "logps/rejected": -31.852075576782227,
-      "loss": 5.3282,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.07687394320964813,
-      "rewards/margins": 0.11593522876501083,
-      "rewards/rejected": -0.039061289280653,
+      "logits/chosen": -3.0258471965789795,
+      "logits/rejected": -3.022930860519409,
+      "logps/chosen": -27.234683990478516,
+      "logps/rejected": -31.9457950592041,
+      "loss": 0.7968,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.17608027160167694,
+      "rewards/margins": 0.3783981502056122,
+      "rewards/rejected": -0.20231786370277405,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 366.0,
+      "grad_norm": 47.25,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -2.813143014907837,
-      "logits/rejected": -2.8078806400299072,
-      "logps/chosen": -27.3418025970459,
-      "logps/rejected": -31.519561767578125,
-      "loss": 4.5462,
+      "logits/chosen": -2.823291063308716,
+      "logits/rejected": -2.818061113357544,
+      "logps/chosen": -27.286752700805664,
+      "logps/rejected": -31.543277740478516,
+      "loss": 0.6353,
       "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.09674083441495895,
-      "rewards/margins": 0.15592411160469055,
-      "rewards/rejected": -0.05918328836560249,
+      "rewards/chosen": 0.3771280348300934,
+      "rewards/margins": 0.6008729934692383,
+      "rewards/rejected": -0.22374498844146729,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 414.0,
+      "grad_norm": 61.0,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -3.1300997734069824,
-      "logits/rejected": -3.1127312183380127,
-      "logps/chosen": -31.788288116455078,
-      "logps/rejected": -29.369943618774414,
-      "loss": 4.1399,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.1406870186328888,
-      "rewards/margins": 0.2187211960554123,
-      "rewards/rejected": -0.0780341774225235,
+      "logits/chosen": -3.141061305999756,
+      "logits/rejected": -3.1238036155700684,
+      "logps/chosen": -31.78842544555664,
+      "logps/rejected": -29.447246551513672,
+      "loss": 0.5473,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.4923074245452881,
+      "rewards/margins": 0.8195418119430542,
+      "rewards/rejected": -0.32723429799079895,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 316.0,
+      "grad_norm": 47.25,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.9419102668762207,
-      "logits/rejected": -2.949079751968384,
-      "logps/chosen": -29.38619613647461,
-      "logps/rejected": -31.473434448242188,
-      "loss": 4.3154,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.10168395191431046,
-      "rewards/margins": 0.1764744520187378,
-      "rewards/rejected": -0.07479049265384674,
+      "logits/chosen": -2.9534718990325928,
+      "logits/rejected": -2.960083484649658,
+      "logps/chosen": -29.32253074645996,
+      "logps/rejected": -31.597339630126953,
+      "loss": 0.5934,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.40046125650405884,
+      "rewards/margins": 0.7489619255065918,
+      "rewards/rejected": -0.34850066900253296,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 330.0,
+      "grad_norm": 51.0,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.790714740753174,
-      "logits/rejected": -2.7880821228027344,
-      "logps/chosen": -29.101242065429688,
-      "logps/rejected": -30.093902587890625,
-      "loss": 4.5857,
+      "logits/chosen": -2.8042290210723877,
+      "logits/rejected": -2.801795244216919,
+      "logps/chosen": -29.153839111328125,
+      "logps/rejected": -30.13584327697754,
+      "loss": 0.6384,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.11372152715921402,
-      "rewards/margins": 0.17886772751808167,
-      "rewards/rejected": -0.06514620780944824,
+      "rewards/chosen": 0.36120903491973877,
+      "rewards/margins": 0.618577778339386,
+      "rewards/rejected": -0.2573687434196472,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 241.0,
+      "grad_norm": 31.125,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.9045286178588867,
-      "logits/rejected": -2.901243209838867,
-      "logps/chosen": -29.739131927490234,
-      "logps/rejected": -28.617401123046875,
-      "loss": 6.9402,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.0895165354013443,
-      "rewards/margins": 0.14149072766304016,
-      "rewards/rejected": -0.051974184811115265,
+      "logits/chosen": -2.9180445671081543,
+      "logits/rejected": -2.914367914199829,
+      "logps/chosen": -29.715240478515625,
+      "logps/rejected": -28.60836410522461,
+      "loss": 0.7637,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.33003363013267517,
+      "rewards/margins": 0.5056184530258179,
+      "rewards/rejected": -0.1755848377943039,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 163.0,
+      "grad_norm": 46.0,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -2.9731786251068115,
-      "logits/rejected": -2.9609808921813965,
-      "logps/chosen": -32.76665496826172,
-      "logps/rejected": -30.380229949951172,
-      "loss": 5.6214,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.18647293746471405,
-      "rewards/margins": 0.17570139467716217,
-      "rewards/rejected": 0.010771543718874454,
+      "logits/chosen": -2.987438201904297,
+      "logits/rejected": -2.97584867477417,
+      "logps/chosen": -32.835445404052734,
+      "logps/rejected": -30.47810935974121,
+      "loss": 0.7257,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.6045005917549133,
+      "rewards/margins": 0.6353167295455933,
+      "rewards/rejected": -0.030816053971648216,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.8129608631134033,
-      "eval_logits/rejected": -2.810295343399048,
-      "eval_logps/chosen": -31.312314987182617,
-      "eval_logps/rejected": -34.8574333190918,
-      "eval_loss": 5.954337120056152,
-      "eval_rewards/accuracies": 0.5685215592384338,
-      "eval_rewards/chosen": -0.005972325801849365,
-      "eval_rewards/margins": 0.02588305063545704,
-      "eval_rewards/rejected": -0.031855374574661255,
-      "eval_runtime": 113.1346,
-      "eval_samples_per_second": 3.032,
+      "eval_logits/chosen": -2.8270444869995117,
+      "eval_logits/rejected": -2.8244259357452393,
+      "eval_logps/chosen": -31.380773544311523,
+      "eval_logps/rejected": -34.95420837402344,
+      "eval_loss": 0.9027834534645081,
+      "eval_rewards/accuracies": 0.5573089718818665,
+      "eval_rewards/chosen": -0.06882722675800323,
+      "eval_rewards/margins": 0.11040809750556946,
+      "eval_rewards/rejected": -0.1792353093624115,
+      "eval_runtime": 113.148,
+      "eval_samples_per_second": 3.031,
       "eval_steps_per_second": 0.38,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 346.0,
+      "grad_norm": 56.25,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.9092488288879395,
-      "logits/rejected": -2.9100584983825684,
-      "logps/chosen": -32.28118133544922,
-      "logps/rejected": -34.049217224121094,
-      "loss": 4.2159,
+      "logits/chosen": -2.9236907958984375,
+      "logits/rejected": -2.9235434532165527,
+      "logps/chosen": -32.24640655517578,
+      "logps/rejected": -34.160125732421875,
+      "loss": 0.5593,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.15044590830802917,
-      "rewards/margins": 0.18799623847007751,
-      "rewards/rejected": -0.03755030035972595,
+      "rewards/chosen": 0.5509027242660522,
+      "rewards/margins": 0.7599622011184692,
+      "rewards/rejected": -0.2090594321489334,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 296.0,
+      "grad_norm": 42.25,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -2.886615037918091,
-      "logits/rejected": -2.902318000793457,
-      "logps/chosen": -29.267192840576172,
-      "logps/rejected": -28.660858154296875,
-      "loss": 4.7348,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.18216967582702637,
-      "rewards/margins": 0.20868726074695587,
-      "rewards/rejected": -0.026517590507864952,
+      "logits/chosen": -2.90382981300354,
+      "logits/rejected": -2.9191505908966064,
+      "logps/chosen": -29.37839698791504,
+      "logps/rejected": -28.82822036743164,
+      "loss": 0.6297,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.5597518682479858,
+      "rewards/margins": 0.7697170972824097,
+      "rewards/rejected": -0.20996519923210144,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 278.0,
+      "grad_norm": 44.5,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -2.9349474906921387,
-      "logits/rejected": -2.9388556480407715,
-      "logps/chosen": -30.56326675415039,
-      "logps/rejected": -31.848459243774414,
-      "loss": 5.1251,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.11771988868713379,
-      "rewards/margins": 0.13578179478645325,
-      "rewards/rejected": -0.01806190237402916,
+      "logits/chosen": -2.9497718811035156,
+      "logits/rejected": -2.954267740249634,
+      "logps/chosen": -30.704570770263672,
+      "logps/rejected": -32.00530242919922,
+      "loss": 0.7476,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.3131088614463806,
+      "rewards/margins": 0.48611265420913696,
+      "rewards/rejected": -0.17300380766391754,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 294.0,
+      "grad_norm": 41.75,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -2.9876904487609863,
-      "logits/rejected": -2.994788408279419,
-      "logps/chosen": -30.00099754333496,
-      "logps/rejected": -30.282485961914062,
-      "loss": 3.5274,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.18078531324863434,
-      "rewards/margins": 0.19023428857326508,
-      "rewards/rejected": -0.009448973461985588,
+      "logits/chosen": -3.004119396209717,
+      "logits/rejected": -3.012087106704712,
+      "logps/chosen": -30.007003784179688,
+      "logps/rejected": -30.454036712646484,
+      "loss": 0.4699,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.6285442113876343,
+      "rewards/margins": 0.7816993594169617,
+      "rewards/rejected": -0.15315520763397217,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 448.0,
+      "grad_norm": 48.25,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.818467617034912,
-      "logits/rejected": -2.8090109825134277,
-      "logps/chosen": -26.312854766845703,
-      "logps/rejected": -29.42915916442871,
-      "loss": 5.123,
+      "logits/chosen": -2.8357105255126953,
+      "logits/rejected": -2.825700283050537,
+      "logps/chosen": -26.410411834716797,
+      "logps/rejected": -29.5693416595459,
+      "loss": 0.719,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.12827233970165253,
-      "rewards/margins": 0.10777638852596283,
-      "rewards/rejected": 0.020495977252721786,
+      "rewards/chosen": 0.3806636929512024,
+      "rewards/margins": 0.40705281496047974,
+      "rewards/rejected": -0.02638910338282585,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 250.0,
+      "grad_norm": 19.25,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -2.800107002258301,
-      "logits/rejected": -2.8200669288635254,
-      "logps/chosen": -28.879364013671875,
-      "logps/rejected": -34.295921325683594,
-      "loss": 3.958,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.19011935591697693,
-      "rewards/margins": 0.21448710560798645,
-      "rewards/rejected": -0.024367772042751312,
+      "logits/chosen": -2.817229747772217,
+      "logits/rejected": -2.837646245956421,
+      "logps/chosen": -28.973642349243164,
+      "logps/rejected": -34.52573013305664,
+      "loss": 0.5428,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.599422812461853,
+      "rewards/margins": 0.8455693125724792,
+      "rewards/rejected": -0.24614660441875458,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 326.0,
+      "grad_norm": 62.25,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -2.942564010620117,
-      "logits/rejected": -2.9483418464660645,
-      "logps/chosen": -29.95509910583496,
-      "logps/rejected": -30.009634017944336,
-      "loss": 3.7773,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.16525179147720337,
-      "rewards/margins": 0.19412627816200256,
-      "rewards/rejected": -0.028874486684799194,
+      "logits/chosen": -2.959622859954834,
+      "logits/rejected": -2.9659740924835205,
+      "logps/chosen": -30.064538955688477,
+      "logps/rejected": -30.272811889648438,
+      "loss": 0.534,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.5017729997634888,
+      "rewards/margins": 0.7870587110519409,
+      "rewards/rejected": -0.28528574109077454,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 214.0,
+      "grad_norm": 31.75,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -2.957038402557373,
-      "logits/rejected": -2.9439315795898438,
-      "logps/chosen": -30.10283851623535,
-      "logps/rejected": -28.607885360717773,
-      "loss": 6.1162,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.12376952171325684,
-      "rewards/margins": 0.11907708644866943,
-      "rewards/rejected": 0.004692415706813335,
+      "logits/chosen": -2.974242687225342,
+      "logits/rejected": -2.960573673248291,
+      "logps/chosen": -30.0527400970459,
+      "logps/rejected": -28.732250213623047,
+      "loss": 0.7261,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.4682597219944,
+      "rewards/margins": 0.5388898253440857,
+      "rewards/rejected": -0.07063011080026627,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 292.0,
+      "grad_norm": 32.25,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.884387969970703,
-      "logits/rejected": -2.8665497303009033,
-      "logps/chosen": -31.607385635375977,
-      "logps/rejected": -30.795541763305664,
-      "loss": 3.528,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.2057834416627884,
-      "rewards/margins": 0.26023000478744507,
-      "rewards/rejected": -0.05444660037755966,
+      "logits/chosen": -2.901106595993042,
+      "logits/rejected": -2.8834261894226074,
+      "logps/chosen": -31.619491577148438,
+      "logps/rejected": -31.00750160217285,
+      "loss": 0.4327,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.7117676734924316,
+      "rewards/margins": 1.0507009029388428,
+      "rewards/rejected": -0.3389332592487335,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 428.0,
+      "grad_norm": 43.5,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -2.887512445449829,
-      "logits/rejected": -2.8846189975738525,
-      "logps/chosen": -31.37001609802246,
-      "logps/rejected": -27.58599281311035,
-      "loss": 4.7461,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.17052757740020752,
-      "rewards/margins": 0.16700942814350128,
-      "rewards/rejected": 0.003518153680488467,
+      "logits/chosen": -2.9032962322235107,
+      "logits/rejected": -2.8999171257019043,
+      "logps/chosen": -31.358158111572266,
+      "logps/rejected": -27.814193725585938,
+      "loss": 0.6308,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.6051477789878845,
+      "rewards/margins": 0.7525756359100342,
+      "rewards/rejected": -0.14742788672447205,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.8048255443573,
-      "eval_logits/rejected": -2.8025119304656982,
-      "eval_logps/chosen": -31.280437469482422,
-      "eval_logps/rejected": -34.820987701416016,
-      "eval_loss": 5.971487522125244,
-      "eval_rewards/accuracies": 0.5747508406639099,
-      "eval_rewards/chosen": 0.00040244098636321723,
-      "eval_rewards/margins": 0.024968715384602547,
-      "eval_rewards/rejected": -0.024566275998950005,
-      "eval_runtime": 112.898,
-      "eval_samples_per_second": 3.038,
-      "eval_steps_per_second": 0.381,
+      "eval_logits/chosen": -2.8207788467407227,
+      "eval_logits/rejected": -2.8184332847595215,
+      "eval_logps/chosen": -31.37415885925293,
+      "eval_logps/rejected": -34.95107650756836,
+      "eval_loss": 0.9011654257774353,
+      "eval_rewards/accuracies": 0.565614640712738,
+      "eval_rewards/chosen": -0.06419765949249268,
+      "eval_rewards/margins": 0.11284750699996948,
+      "eval_rewards/rejected": -0.17704515159130096,
+      "eval_runtime": 113.1332,
+      "eval_samples_per_second": 3.032,
+      "eval_steps_per_second": 0.38,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 408.0,
+      "grad_norm": 52.0,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -2.767286777496338,
-      "logits/rejected": -2.7845776081085205,
-      "logps/chosen": -28.39835548400879,
-      "logps/rejected": -30.915149688720703,
-      "loss": 4.0113,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.1662132441997528,
-      "rewards/margins": 0.20284172892570496,
-      "rewards/rejected": -0.03662847727537155,
+      "logits/chosen": -2.7831432819366455,
+      "logits/rejected": -2.801016092300415,
+      "logps/chosen": -28.50387191772461,
+      "logps/rejected": -31.167369842529297,
+      "loss": 0.4507,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.5078839659690857,
+      "rewards/margins": 0.8126392364501953,
+      "rewards/rejected": -0.30475518107414246,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 186.0,
+      "grad_norm": 31.625,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -3.013503074645996,
-      "logits/rejected": -2.9991822242736816,
-      "logps/chosen": -29.416873931884766,
-      "logps/rejected": -28.512454986572266,
-      "loss": 3.4113,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.1589430421590805,
-      "rewards/margins": 0.19767849147319794,
-      "rewards/rejected": -0.03873545676469803,
+      "logits/chosen": -3.029501438140869,
+      "logits/rejected": -3.0155110359191895,
+      "logps/chosen": -29.30743408203125,
+      "logps/rejected": -28.680965423583984,
+      "loss": 0.4592,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.6329048871994019,
+      "rewards/margins": 0.8864381909370422,
+      "rewards/rejected": -0.25353336334228516,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 159.0,
+      "grad_norm": 17.0,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.928607702255249,
-      "logits/rejected": -2.911008358001709,
-      "logps/chosen": -27.41733741760254,
-      "logps/rejected": -30.696624755859375,
-      "loss": 3.3907,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.188947394490242,
-      "rewards/margins": 0.24994292855262756,
-      "rewards/rejected": -0.060995571315288544,
+      "logits/chosen": -2.943554401397705,
+      "logits/rejected": -2.925654888153076,
+      "logps/chosen": -27.233322143554688,
+      "logps/rejected": -30.869421005249023,
+      "loss": 0.4325,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.7901242971420288,
+      "rewards/margins": 1.1245664358139038,
+      "rewards/rejected": -0.33444222807884216,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 342.0,
+      "grad_norm": 53.5,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -3.143974542617798,
-      "logits/rejected": -3.1502108573913574,
-      "logps/chosen": -30.86354637145996,
-      "logps/rejected": -33.04155349731445,
-      "loss": 4.0138,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.15513095259666443,
-      "rewards/margins": 0.22424118220806122,
-      "rewards/rejected": -0.06911022216081619,
+      "logits/chosen": -3.1611580848693848,
+      "logits/rejected": -3.167454242706299,
+      "logps/chosen": -30.857864379882812,
+      "logps/rejected": -33.212623596191406,
+      "loss": 0.4854,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.546938955783844,
+      "rewards/margins": 0.908575177192688,
+      "rewards/rejected": -0.3616361618041992,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 200.0,
+      "grad_norm": 40.0,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -3.021571159362793,
-      "logits/rejected": -3.025432586669922,
-      "logps/chosen": -29.817678451538086,
-      "logps/rejected": -31.675588607788086,
-      "loss": 4.188,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.19784057140350342,
-      "rewards/margins": 0.23029597103595734,
-      "rewards/rejected": -0.03245542570948601,
+      "logits/chosen": -3.037867546081543,
+      "logits/rejected": -3.040760040283203,
+      "logps/chosen": -29.823516845703125,
+      "logps/rejected": -31.82476806640625,
+      "loss": 0.5086,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.6883540153503418,
+      "rewards/margins": 0.9063750505447388,
+      "rewards/rejected": -0.21802091598510742,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 286.0,
+      "grad_norm": 41.5,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -2.86122727394104,
-      "logits/rejected": -2.8630313873291016,
-      "logps/chosen": -27.54501724243164,
-      "logps/rejected": -29.912155151367188,
-      "loss": 3.5611,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.2209421843290329,
-      "rewards/margins": 0.23718467354774475,
-      "rewards/rejected": -0.01624247059226036,
+      "logits/chosen": -2.877352476119995,
+      "logits/rejected": -2.878804922103882,
+      "logps/chosen": -27.442968368530273,
+      "logps/rejected": -30.033910751342773,
+      "loss": 0.4586,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.8447319865226746,
+      "rewards/margins": 0.986809253692627,
+      "rewards/rejected": -0.1420772522687912,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 324.0,
+      "grad_norm": 66.5,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.935516357421875,
-      "logits/rejected": -2.9337944984436035,
-      "logps/chosen": -29.81747055053711,
-      "logps/rejected": -32.104270935058594,
-      "loss": 3.9741,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.1938357651233673,
-      "rewards/margins": 0.2290809601545334,
-      "rewards/rejected": -0.03524521738290787,
+      "logits/chosen": -2.9520726203918457,
+      "logits/rejected": -2.949944257736206,
+      "logps/chosen": -29.80544090270996,
+      "logps/rejected": -32.19409942626953,
+      "loss": 0.5126,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.6868453025817871,
+      "rewards/margins": 0.873086154460907,
+      "rewards/rejected": -0.1862407922744751,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 222.0,
+      "grad_norm": 49.5,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -2.8935000896453857,
-      "logits/rejected": -2.8769774436950684,
-      "logps/chosen": -28.101531982421875,
-      "logps/rejected": -28.089923858642578,
-      "loss": 3.3899,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.1687012016773224,
-      "rewards/margins": 0.20876184105873108,
-      "rewards/rejected": -0.0400606133043766,
+      "logits/chosen": -2.908616542816162,
+      "logits/rejected": -2.890885829925537,
+      "logps/chosen": -28.092891693115234,
+      "logps/rejected": -28.3051700592041,
+      "loss": 0.4503,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.5965021252632141,
+      "rewards/margins": 0.8873867988586426,
+      "rewards/rejected": -0.290884792804718,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 4.931960544338474,
-      "train_runtime": 2724.0851,
-      "train_samples_per_second": 1.13,
-      "train_steps_per_second": 0.141
+      "train_loss": 0.6684874992866021,
+      "train_runtime": 2718.5428,
+      "train_samples_per_second": 1.133,
+      "train_steps_per_second": 0.142
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64fcaf1cde7f01f2ea4eaeb303540292c988722951db76d85c3aebe520f3b780
+oid sha256:595a4589f4c882fa1c1b4a634e18c5da9619fbb1a37826997cc815b9179e2a02
 size 5176