Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NorLLM-AI/NorMistral-7B
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: norllm-ai-normistral-7b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # norllm-ai-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/norllm-ai-normistral-7b-sft-qlora](https://huggingface.co/data/norllm-ai-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NorLLM-AI/NorMistral-7B](https://huggingface.co/NorLLM-AI/NorMistral-7B) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4935
-- Rewards/chosen: 0.0418
-- Rewards/rejected: 0.0072
-- Rewards/accuracies: 0.5602
-- Rewards/margins: 0.0346
-- Logps/rejected: -34.6891
-- Logps/chosen: -31.2302
-- Logits/rejected: -2.8078
-- Logits/chosen: -2.8103
+- Loss: 1.0892
+- Rewards/chosen: 0.0295
+- Rewards/rejected: -0.0000
+- Rewards/accuracies: 0.5457
+- Rewards/margins: 0.0295
+- Logps/rejected: -34.6982
+- Logps/chosen: -31.2235
+- Logits/rejected: -2.8015
+- Logits/chosen: -2.8038
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 1.0237        | 0.26  | 100  | 0.4815          | 0.0235         | -0.0157          | 0.5486             | 0.0392          | -34.7178       | -31.2531     | -2.8136         | -2.8161       |
-| 2.9692        | 0.52  | 200  | 0.4973          | 0.0315         | -0.0118          | 0.5631             | 0.0433          | -34.7129       | -31.2431     | -2.8099         | -2.8121       |
-| 0.9535        | 0.78  | 300  | 0.4705          | 0.0616         | 0.0107           | 0.5370             | 0.0508          | -34.6847       | -31.2055     | -2.8071         | -2.8098       |
+| 1.2624        | 0.26  | 100  | 1.0410          | 0.0177         | -0.0236          | 0.5718             | 0.0413          | -34.7453       | -31.2470     | -2.8153         | -2.8178       |
+| 2.7933        | 0.52  | 200  | 1.0914          | 0.0137         | -0.0134          | 0.5224             | 0.0271          | -34.7250       | -31.2550     | -2.8053         | -2.8078       |
+| 1.1004        | 0.78  | 300  | 1.0892          | 0.0295         | -0.0000          | 0.5457             | 0.0295          | -34.6982       | -31.2235     | -2.8015         | -2.8038       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e875a9f24872c76e27cd5cea85752345b51ee7bd4950f97e896214cfd8861dcf
+oid sha256:157a8738608b33d48cac1b9db15448852f0cac0e2045a5f15f74877c53bbfae0
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.810283660888672,
-    "eval_logits/rejected": -2.8077523708343506,
-    "eval_logps/chosen": -31.230178833007812,
-    "eval_logps/rejected": -34.68911361694336,
-    "eval_loss": 0.49349722266197205,
-    "eval_rewards/accuracies": 0.560215950012207,
-    "eval_rewards/chosen": 0.04181605204939842,
-    "eval_rewards/margins": 0.03458160161972046,
-    "eval_rewards/rejected": 0.007234453689306974,
-    "eval_runtime": 112.8053,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.041,
-    "eval_steps_per_second": 0.381,
-    "train_loss": 1.1209261448352368,
-    "train_runtime": 2723.1709,
+    "train_loss": 1.4646434090354226,
+    "train_runtime": 2724.3946,
     "train_samples": 3079,
-    "train_samples_per_second": 1.131,
+    "train_samples_per_second": 1.13,
     "train_steps_per_second": 0.141
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 1.1209261448352368,
-    "train_runtime": 2723.1709,
+    "train_loss": 1.4646434090354226,
+    "train_runtime": 2724.3946,
     "train_samples": 3079,
-    "train_samples_per_second": 1.131,
+    "train_samples_per_second": 1.13,
     "train_steps_per_second": 0.141
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 99.0,
+      "grad_norm": 158.0,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": -2.7358343601226807,
       "logits/rejected": -2.7480404376983643,
       "logps/chosen": -27.35565757751465,
       "logps/rejected": -21.06114387512207,
-      "loss": 0.3906,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,629 +25,629 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 161.0,
+      "grad_norm": 272.0,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -3.0094523429870605,
-      "logits/rejected": -2.998090982437134,
-      "logps/chosen": -33.19883728027344,
-      "logps/rejected": -31.973581314086914,
-      "loss": 0.4457,
-      "rewards/accuracies": 0.4166666567325592,
-      "rewards/chosen": -0.014314251020550728,
-      "rewards/margins": -0.006575056351721287,
-      "rewards/rejected": -0.007739194668829441,
+      "logits/chosen": -3.010078191757202,
+      "logits/rejected": -2.9986138343811035,
+      "logps/chosen": -33.18549346923828,
+      "logps/rejected": -31.97589111328125,
+      "loss": 1.04,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": -0.00227526156231761,
+      "rewards/margins": 0.003717611311003566,
+      "rewards/rejected": -0.005992872640490532,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 128.0,
+      "grad_norm": 185.0,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -2.8992228507995605,
-      "logits/rejected": -2.8941798210144043,
-      "logps/chosen": -32.46299362182617,
-      "logps/rejected": -28.942882537841797,
-      "loss": 0.4894,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.009680476039648056,
-      "rewards/margins": -0.021505311131477356,
-      "rewards/rejected": 0.031185787171125412,
+      "logits/chosen": -2.899261713027954,
+      "logits/rejected": -2.894345998764038,
+      "logps/chosen": -32.464820861816406,
+      "logps/rejected": -28.955495834350586,
+      "loss": 1.0912,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 0.005139008164405823,
+      "rewards/margins": -0.008046457543969154,
+      "rewards/rejected": 0.013185465708374977,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 129.0,
+      "grad_norm": 178.0,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -3.097111225128174,
-      "logits/rejected": -3.109170436859131,
-      "logps/chosen": -32.810890197753906,
-      "logps/rejected": -30.153121948242188,
-      "loss": 0.4421,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.07823446393013,
-      "rewards/margins": 0.04388953000307083,
-      "rewards/rejected": 0.03434494137763977,
+      "logits/chosen": -3.0970449447631836,
+      "logits/rejected": -3.1092135906219482,
+      "logps/chosen": -32.8364143371582,
+      "logps/rejected": -30.134708404541016,
+      "loss": 1.0707,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.03613559529185295,
+      "rewards/margins": 0.005463692359626293,
+      "rewards/rejected": 0.030671900138258934,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 200.0,
+      "grad_norm": 218.0,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.86364483833313,
-      "logits/rejected": -2.853883981704712,
-      "logps/chosen": -31.620784759521484,
-      "logps/rejected": -32.36823654174805,
-      "loss": 0.7104,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.1676289290189743,
-      "rewards/margins": 0.17625470459461212,
-      "rewards/rejected": -0.008625739254057407,
+      "logits/chosen": -2.8639309406280518,
+      "logits/rejected": -2.8546805381774902,
+      "logps/chosen": -31.568679809570312,
+      "logps/rejected": -32.38216781616211,
+      "loss": 1.1343,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.13082042336463928,
+      "rewards/margins": 0.14317770302295685,
+      "rewards/rejected": -0.012357300147414207,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 229.0,
+      "grad_norm": 226.0,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -2.886505603790283,
-      "logits/rejected": -2.884448528289795,
-      "logps/chosen": -29.601898193359375,
-      "logps/rejected": -30.147045135498047,
-      "loss": 0.8433,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.11060991138219833,
-      "rewards/margins": 0.15709055960178375,
-      "rewards/rejected": -0.04648064821958542,
+      "logits/chosen": -2.885357618331909,
+      "logits/rejected": -2.883885145187378,
+      "logps/chosen": -29.530975341796875,
+      "logps/rejected": -30.174474716186523,
+      "loss": 1.1329,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.10459370911121368,
+      "rewards/margins": 0.14735892415046692,
+      "rewards/rejected": -0.04276522248983383,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 137.0,
+      "grad_norm": 210.0,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -2.9212472438812256,
-      "logits/rejected": -2.921980381011963,
-      "logps/chosen": -30.149127960205078,
-      "logps/rejected": -28.14251136779785,
-      "loss": 0.5196,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.01548430323600769,
-      "rewards/margins": 0.13397768139839172,
-      "rewards/rejected": -0.1494619995355606,
+      "logits/chosen": -2.919003486633301,
+      "logits/rejected": -2.921109199523926,
+      "logps/chosen": -30.042144775390625,
+      "logps/rejected": -28.137714385986328,
+      "loss": 0.8828,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.04381434619426727,
+      "rewards/margins": 0.13482996821403503,
+      "rewards/rejected": -0.09101562947034836,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 1104.0,
+      "grad_norm": 1168.0,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -3.0032145977020264,
-      "logits/rejected": -3.010529041290283,
-      "logps/chosen": -29.345632553100586,
-      "logps/rejected": -31.078609466552734,
-      "loss": 1.3076,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.0517343208193779,
-      "rewards/margins": 0.09771490097045898,
-      "rewards/rejected": -0.1494491994380951,
+      "logits/chosen": -3.001248598098755,
+      "logits/rejected": -3.008502244949341,
+      "logps/chosen": -29.325231552124023,
+      "logps/rejected": -31.089244842529297,
+      "loss": 1.7989,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.022131171077489853,
+      "rewards/margins": 0.07659248262643814,
+      "rewards/rejected": -0.0987236499786377,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 186.0,
+      "grad_norm": 243.0,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -2.816901922225952,
-      "logits/rejected": -2.832296848297119,
-      "logps/chosen": -29.427043914794922,
-      "logps/rejected": -29.738506317138672,
-      "loss": 0.6608,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0909457728266716,
-      "rewards/margins": 0.14232560992240906,
-      "rewards/rejected": -0.051379840821027756,
+      "logits/chosen": -2.8158023357391357,
+      "logits/rejected": -2.8314785957336426,
+      "logps/chosen": -29.39749526977539,
+      "logps/rejected": -29.807388305664062,
+      "loss": 1.0484,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.07161647081375122,
+      "rewards/margins": 0.13816848397254944,
+      "rewards/rejected": -0.06655202060937881,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 216.0,
+      "grad_norm": 376.0,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -2.902531623840332,
-      "logits/rejected": -2.884403705596924,
-      "logps/chosen": -32.63992691040039,
-      "logps/rejected": -30.051746368408203,
-      "loss": 3.8065,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.15720424056053162,
-      "rewards/margins": 0.18128542602062225,
-      "rewards/rejected": -0.024081196635961533,
+      "logits/chosen": -2.9029974937438965,
+      "logits/rejected": -2.8848369121551514,
+      "logps/chosen": -32.61262893676758,
+      "logps/rejected": -30.10404396057129,
+      "loss": 5.5978,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.11190228164196014,
+      "rewards/margins": 0.1531006246805191,
+      "rewards/rejected": -0.04119832441210747,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 162.0,
+      "grad_norm": 161.0,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -3.0053141117095947,
-      "logits/rejected": -3.0066592693328857,
-      "logps/chosen": -31.79007911682129,
-      "logps/rejected": -30.816768646240234,
-      "loss": 1.0237,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.2134353667497635,
-      "rewards/margins": 0.21390971541404724,
-      "rewards/rejected": -0.000474362081149593,
+      "logits/chosen": -3.008094310760498,
+      "logits/rejected": -3.008415699005127,
+      "logps/chosen": -31.737361907958984,
+      "logps/rejected": -30.788349151611328,
+      "loss": 1.2624,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.1597568690776825,
+      "rewards/margins": 0.14584486186504364,
+      "rewards/rejected": 0.013912007212638855,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.8161001205444336,
-      "eval_logits/rejected": -2.8135948181152344,
-      "eval_logps/chosen": -31.253108978271484,
-      "eval_logps/rejected": -34.717830657958984,
-      "eval_loss": 0.4814997613430023,
-      "eval_rewards/accuracies": 0.5485880374908447,
-      "eval_rewards/chosen": 0.023475930094718933,
-      "eval_rewards/margins": 0.039213381707668304,
-      "eval_rewards/rejected": -0.015737449750304222,
-      "eval_runtime": 113.131,
-      "eval_samples_per_second": 3.032,
-      "eval_steps_per_second": 0.38,
+      "eval_logits/chosen": -2.817828416824341,
+      "eval_logits/rejected": -2.8153038024902344,
+      "eval_logps/chosen": -31.246990203857422,
+      "eval_logps/rejected": -34.745269775390625,
+      "eval_loss": 1.0410326719284058,
+      "eval_rewards/accuracies": 0.5718438625335693,
+      "eval_rewards/chosen": 0.01773122325539589,
+      "eval_rewards/margins": 0.04128904268145561,
+      "eval_rewards/rejected": -0.023557817563414574,
+      "eval_runtime": 113.418,
+      "eval_samples_per_second": 3.024,
+      "eval_steps_per_second": 0.379,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 156.0,
+      "grad_norm": 225.0,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -2.962667942047119,
-      "logits/rejected": -2.9380180835723877,
-      "logps/chosen": -31.729461669921875,
-      "logps/rejected": -31.084197998046875,
-      "loss": 1.6614,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.38986626267433167,
-      "rewards/margins": 0.33014416694641113,
-      "rewards/rejected": 0.05972205847501755,
+      "logits/chosen": -2.9651312828063965,
+      "logits/rejected": -2.941673994064331,
+      "logps/chosen": -31.744415283203125,
+      "logps/rejected": -31.186565399169922,
+      "loss": 1.2513,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.2361893206834793,
+      "rewards/margins": 0.25004786252975464,
+      "rewards/rejected": -0.013858571648597717,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 100.5,
+      "grad_norm": 136.0,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -3.0431807041168213,
-      "logits/rejected": -3.072758197784424,
-      "logps/chosen": -28.905542373657227,
-      "logps/rejected": -34.141502380371094,
-      "loss": 0.822,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.20668914914131165,
-      "rewards/margins": 0.20185013115406036,
-      "rewards/rejected": 0.004838997032493353,
+      "logits/chosen": -3.044619083404541,
+      "logits/rejected": -3.074309825897217,
+      "logps/chosen": -28.8009033203125,
+      "logps/rejected": -34.08342742919922,
+      "loss": 1.442,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.18149998784065247,
+      "rewards/margins": 0.14943717420101166,
+      "rewards/rejected": 0.03206281363964081,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 73.5,
+      "grad_norm": 106.0,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.743009090423584,
-      "logits/rejected": -2.7388014793395996,
-      "logps/chosen": -28.7690486907959,
-      "logps/rejected": -30.156269073486328,
-      "loss": 0.7399,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.16450510919094086,
-      "rewards/margins": 0.1824747771024704,
-      "rewards/rejected": -0.01796967163681984,
+      "logits/chosen": -2.7447521686553955,
+      "logits/rejected": -2.7404866218566895,
+      "logps/chosen": -28.633804321289062,
+      "logps/rejected": -30.120105743408203,
+      "loss": 1.5003,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.17043739557266235,
+      "rewards/margins": 0.16358675062656403,
+      "rewards/rejected": 0.0068506537936627865,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 197.0,
+      "grad_norm": 180.0,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -3.015079975128174,
-      "logits/rejected": -3.0121235847473145,
-      "logps/chosen": -27.263751983642578,
-      "logps/rejected": -31.766677856445312,
-      "loss": 0.8956,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.17797937989234924,
-      "rewards/margins": 0.2659076154232025,
-      "rewards/rejected": -0.08792825043201447,
+      "logits/chosen": -3.017597198486328,
+      "logits/rejected": -3.0148470401763916,
+      "logps/chosen": -27.30155372619629,
+      "logps/rejected": -31.824840545654297,
+      "loss": 1.328,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0923347920179367,
+      "rewards/margins": 0.17637018859386444,
+      "rewards/rejected": -0.08403542637825012,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 126.0,
+      "grad_norm": 208.0,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -2.807605028152466,
-      "logits/rejected": -2.8025310039520264,
-      "logps/chosen": -27.548370361328125,
-      "logps/rejected": -31.246013641357422,
-      "loss": 0.6864,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.22170917689800262,
-      "rewards/margins": 0.23960641026496887,
-      "rewards/rejected": -0.01789720356464386,
+      "logits/chosen": -2.8109421730041504,
+      "logits/rejected": -2.8055999279022217,
+      "logps/chosen": -27.452152252197266,
+      "logps/rejected": -31.32564353942871,
+      "loss": 1.0556,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.18667848408222198,
+      "rewards/margins": 0.2376781404018402,
+      "rewards/rejected": -0.05099965259432793,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 97.0,
+      "grad_norm": 227.0,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -3.123861312866211,
-      "logits/rejected": -3.10616397857666,
-      "logps/chosen": -32.041046142578125,
-      "logps/rejected": -28.996301651000977,
-      "loss": 1.2758,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.36053839325904846,
-      "rewards/margins": 0.3737615942955017,
-      "rewards/rejected": -0.013223287649452686,
+      "logits/chosen": -3.1248366832733154,
+      "logits/rejected": -3.1071696281433105,
+      "logps/chosen": -31.928054809570312,
+      "logps/rejected": -29.134618759155273,
+      "loss": 1.9603,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.2818322777748108,
+      "rewards/margins": 0.3592532277107239,
+      "rewards/rejected": -0.07742089033126831,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 150.0,
+      "grad_norm": 126.0,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.9405550956726074,
-      "logits/rejected": -2.947495222091675,
-      "logps/chosen": -29.35744857788086,
-      "logps/rejected": -31.04239845275879,
-      "loss": 1.0844,
+      "logits/chosen": -2.939929485321045,
+      "logits/rejected": -2.946718692779541,
+      "logps/chosen": -29.42165756225586,
+      "logps/rejected": -31.19052505493164,
+      "loss": 1.5725,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.42973223328590393,
-      "rewards/margins": 0.3840641677379608,
-      "rewards/rejected": 0.045668020844459534,
+      "rewards/chosen": 0.2364785224199295,
+      "rewards/margins": 0.28199905157089233,
+      "rewards/rejected": -0.04552050679922104,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 87.5,
+      "grad_norm": 104.0,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.7875921726226807,
-      "logits/rejected": -2.784825086593628,
-      "logps/chosen": -29.229150772094727,
-      "logps/rejected": -29.77825927734375,
-      "loss": 0.7375,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.35255879163742065,
-      "rewards/margins": 0.36062949895858765,
-      "rewards/rejected": -0.008070724084973335,
+      "logits/chosen": -2.7864370346069336,
+      "logits/rejected": -2.7840218544006348,
+      "logps/chosen": -29.19491958618164,
+      "logps/rejected": -29.839929580688477,
+      "loss": 1.2802,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.2374652922153473,
+      "rewards/margins": 0.27334436774253845,
+      "rewards/rejected": -0.03587912395596504,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 314.0,
+      "grad_norm": 228.0,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.905453681945801,
-      "logits/rejected": -2.901573896408081,
-      "logps/chosen": -29.90896224975586,
-      "logps/rejected": -28.3870906829834,
-      "loss": 1.6507,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.22220328450202942,
-      "rewards/margins": 0.24585184454917908,
-      "rewards/rejected": -0.023648545145988464,
+      "logits/chosen": -2.9004275798797607,
+      "logits/rejected": -2.8972795009613037,
+      "logps/chosen": -29.824024200439453,
+      "logps/rejected": -28.415607452392578,
+      "loss": 2.0411,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.18134601414203644,
+      "rewards/margins": 0.2103847712278366,
+      "rewards/rejected": -0.02903876267373562,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 320.0,
+      "grad_norm": 252.0,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -2.97483491897583,
-      "logits/rejected": -2.9631428718566895,
-      "logps/chosen": -33.094482421875,
-      "logps/rejected": -30.27695655822754,
-      "loss": 2.9692,
+      "logits/chosen": -2.970046281814575,
+      "logits/rejected": -2.957967519760132,
+      "logps/chosen": -33.08484649658203,
+      "logps/rejected": -30.334461212158203,
+      "loss": 2.7933,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.48363131284713745,
-      "rewards/margins": 0.35792380571365356,
-      "rewards/rejected": 0.12570755183696747,
+      "rewards/chosen": 0.3070868253707886,
+      "rewards/margins": 0.25727197527885437,
+      "rewards/rejected": 0.049814820289611816,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.812093496322632,
-      "eval_logits/rejected": -2.809893846511841,
-      "eval_logps/chosen": -31.243083953857422,
-      "eval_logps/rejected": -34.712913513183594,
-      "eval_loss": 0.4973182678222656,
-      "eval_rewards/accuracies": 0.5631229281425476,
-      "eval_rewards/chosen": 0.03149374574422836,
-      "eval_rewards/margins": 0.04329964146018028,
-      "eval_rewards/rejected": -0.011805894784629345,
-      "eval_runtime": 113.1074,
-      "eval_samples_per_second": 3.033,
+      "eval_logits/chosen": -2.8077902793884277,
+      "eval_logits/rejected": -2.805346965789795,
+      "eval_logps/chosen": -31.255006790161133,
+      "eval_logps/rejected": -34.7249755859375,
+      "eval_loss": 1.091357707977295,
+      "eval_rewards/accuracies": 0.5224252939224243,
+      "eval_rewards/chosen": 0.013722050935029984,
+      "eval_rewards/margins": 0.02713143080472946,
+      "eval_rewards/rejected": -0.013409379869699478,
+      "eval_runtime": 113.1622,
+      "eval_samples_per_second": 3.031,
       "eval_steps_per_second": 0.38,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 112.5,
+      "grad_norm": 222.0,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.9090988636016846,
-      "logits/rejected": -2.9100005626678467,
-      "logps/chosen": -32.562644958496094,
-      "logps/rejected": -33.86018371582031,
-      "loss": 1.264,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.37660837173461914,
-      "rewards/margins": 0.375582754611969,
-      "rewards/rejected": 0.0010256364475935698,
+      "logits/chosen": -2.905710458755493,
+      "logits/rejected": -2.9065823554992676,
+      "logps/chosen": -32.491180419921875,
+      "logps/rejected": -33.91350173950195,
+      "loss": 1.5755,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.27111583948135376,
+      "rewards/margins": 0.2971333861351013,
+      "rewards/rejected": -0.02601751685142517,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 204.0,
+      "grad_norm": 206.0,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -2.8871023654937744,
-      "logits/rejected": -2.90246844291687,
-      "logps/chosen": -29.662246704101562,
-      "logps/rejected": -28.47257423400879,
-      "loss": 1.4961,
+      "logits/chosen": -2.881995677947998,
+      "logits/rejected": -2.8976504802703857,
+      "logps/chosen": -29.550067901611328,
+      "logps/rejected": -28.476770401000977,
+      "loss": 1.9387,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.4126364588737488,
-      "rewards/margins": 0.3680770695209503,
-      "rewards/rejected": 0.04455941915512085,
+      "rewards/chosen": 0.3139877915382385,
+      "rewards/margins": 0.2882370054721832,
+      "rewards/rejected": 0.025750819593667984,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 87.5,
+      "grad_norm": 127.0,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -2.9373838901519775,
-      "logits/rejected": -2.941819667816162,
-      "logps/chosen": -30.841754913330078,
-      "logps/rejected": -31.728525161743164,
-      "loss": 0.7905,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.2480934113264084,
-      "rewards/margins": 0.2243928462266922,
-      "rewards/rejected": 0.02370055951178074,
+      "logits/chosen": -2.9315438270568848,
+      "logits/rejected": -2.936497926712036,
+      "logps/chosen": -30.702800750732422,
+      "logps/rejected": -31.729888916015625,
+      "loss": 1.2017,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.22453579306602478,
+      "rewards/margins": 0.2104046642780304,
+      "rewards/rejected": 0.014131096191704273,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 73.0,
+      "grad_norm": 105.0,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -2.987827777862549,
-      "logits/rejected": -2.9960155487060547,
-      "logps/chosen": -30.307302474975586,
-      "logps/rejected": -30.214122772216797,
-      "loss": 0.9428,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.4780976176261902,
-      "rewards/margins": 0.46120333671569824,
-      "rewards/rejected": 0.016894282773137093,
+      "logits/chosen": -2.9826419353485107,
+      "logits/rejected": -2.9898297786712646,
+      "logps/chosen": -30.16364097595215,
+      "logps/rejected": -30.215124130249023,
+      "loss": 0.5978,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.37064167857170105,
+      "rewards/margins": 0.3605824410915375,
+      "rewards/rejected": 0.010059216991066933,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 296.0,
+      "grad_norm": 328.0,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.818535566329956,
-      "logits/rejected": -2.8091442584991455,
-      "logps/chosen": -26.581512451171875,
-      "logps/rejected": -29.30862808227539,
-      "loss": 1.3618,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.2981645464897156,
-      "rewards/margins": 0.11975376307964325,
-      "rewards/rejected": 0.17841079831123352,
+      "logits/chosen": -2.813474178314209,
+      "logits/rejected": -2.803966999053955,
+      "logps/chosen": -26.481496810913086,
+      "logps/rejected": -29.306514739990234,
+      "loss": 1.5582,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.23635974526405334,
+      "rewards/margins": 0.12379670143127441,
+      "rewards/rejected": 0.11256307363510132,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 127.5,
+      "grad_norm": 144.0,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -2.802452802658081,
-      "logits/rejected": -2.8227944374084473,
-      "logps/chosen": -29.27554702758789,
-      "logps/rejected": -34.11919403076172,
-      "loss": 1.2344,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.4435352385044098,
-      "rewards/margins": 0.3996209502220154,
-      "rewards/rejected": 0.043914251029491425,
+      "logits/chosen": -2.795308828353882,
+      "logits/rejected": -2.8167600631713867,
+      "logps/chosen": -29.191823959350586,
+      "logps/rejected": -34.18637466430664,
+      "loss": 0.9343,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.31906968355178833,
+      "rewards/margins": 0.3252100348472595,
+      "rewards/rejected": -0.006140357349067926,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 64.5,
+      "grad_norm": 94.0,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -2.944246768951416,
-      "logits/rejected": -2.9497275352478027,
-      "logps/chosen": -30.331872940063477,
-      "logps/rejected": -29.763620376586914,
-      "loss": 0.9017,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.3595869243144989,
-      "rewards/margins": 0.27827444672584534,
-      "rewards/rejected": 0.08131249994039536,
+      "logits/chosen": -2.939216136932373,
+      "logits/rejected": -2.944835662841797,
+      "logps/chosen": -30.2105655670166,
+      "logps/rejected": -29.839502334594727,
+      "loss": 1.1661,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2853953242301941,
+      "rewards/margins": 0.2725152373313904,
+      "rewards/rejected": 0.012880122289061546,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 87.0,
+      "grad_norm": 82.0,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -2.9589145183563232,
-      "logits/rejected": -2.9460854530334473,
-      "logps/chosen": -30.4505615234375,
-      "logps/rejected": -28.466442108154297,
-      "loss": 1.0356,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.2169012576341629,
-      "rewards/margins": 0.08497779071331024,
-      "rewards/rejected": 0.13192343711853027,
+      "logits/chosen": -2.9545788764953613,
+      "logits/rejected": -2.9408528804779053,
+      "logps/chosen": -30.2893123626709,
+      "logps/rejected": -28.4702205657959,
+      "loss": 1.683,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.21618542075157166,
+      "rewards/margins": 0.13562074303627014,
+      "rewards/rejected": 0.08056464046239853,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 137.0,
+      "grad_norm": 182.0,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.8869829177856445,
-      "logits/rejected": -2.868814706802368,
-      "logps/chosen": -32.20147705078125,
-      "logps/rejected": -30.42227554321289,
-      "loss": 1.5568,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.3478601574897766,
-      "rewards/margins": 0.2670307755470276,
-      "rewards/rejected": 0.08082933723926544,
+      "logits/chosen": -2.8815903663635254,
+      "logits/rejected": -2.8631927967071533,
+      "logps/chosen": -32.01317596435547,
+      "logps/rejected": -30.49240493774414,
+      "loss": 1.6127,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.3115643262863159,
+      "rewards/margins": 0.29611119627952576,
+      "rewards/rejected": 0.015453122556209564,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 362.0,
+      "grad_norm": 350.0,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -2.8908376693725586,
-      "logits/rejected": -2.887605905532837,
-      "logps/chosen": -31.88857078552246,
-      "logps/rejected": -27.47490882873535,
-      "loss": 0.9535,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.2672681212425232,
-      "rewards/margins": 0.16432540118694305,
-      "rewards/rejected": 0.10294272005558014,
+      "logits/chosen": -2.886591911315918,
+      "logits/rejected": -2.8827834129333496,
+      "logps/chosen": -31.685373306274414,
+      "logps/rejected": -27.486093521118164,
+      "loss": 1.1004,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.2686397135257721,
+      "rewards/margins": 0.209893137216568,
+      "rewards/rejected": 0.0587465763092041,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.8098394870758057,
-      "eval_logits/rejected": -2.807095527648926,
-      "eval_logps/chosen": -31.205495834350586,
-      "eval_logps/rejected": -34.68474197387695,
-      "eval_loss": 0.47054800391197205,
-      "eval_rewards/accuracies": 0.5369601845741272,
-      "eval_rewards/chosen": 0.0615648552775383,
-      "eval_rewards/margins": 0.050835300236940384,
-      "eval_rewards/rejected": 0.010729559697210789,
-      "eval_runtime": 112.9246,
-      "eval_samples_per_second": 3.037,
+      "eval_logits/chosen": -2.8038344383239746,
+      "eval_logits/rejected": -2.8014862537384033,
+      "eval_logps/chosen": -31.223527908325195,
+      "eval_logps/rejected": -34.69816970825195,
+      "eval_loss": 1.0891914367675781,
+      "eval_rewards/accuracies": 0.5456810593605042,
+      "eval_rewards/chosen": 0.02946147881448269,
+      "eval_rewards/margins": 0.02946743369102478,
+      "eval_rewards/rejected": -5.9524941207200754e-06,
+      "eval_runtime": 112.8601,
+      "eval_samples_per_second": 3.039,
       "eval_steps_per_second": 0.381,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 139.0,
+      "grad_norm": 178.0,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -2.769951343536377,
-      "logits/rejected": -2.7882049083709717,
-      "logps/chosen": -28.826038360595703,
-      "logps/rejected": -30.570903778076172,
-      "loss": 1.2033,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.322704941034317,
-      "rewards/margins": 0.19382169842720032,
-      "rewards/rejected": 0.1288832128047943,
+      "logits/chosen": -2.7640256881713867,
+      "logits/rejected": -2.7830758094787598,
+      "logps/chosen": -28.689090728759766,
+      "logps/rejected": -30.597253799438477,
+      "loss": 1.4386,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.27016550302505493,
+      "rewards/margins": 0.20278863608837128,
+      "rewards/rejected": 0.06737686693668365,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 142.0,
+      "grad_norm": 119.5,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -3.0174190998077393,
-      "logits/rejected": -3.002908706665039,
-      "logps/chosen": -29.645282745361328,
-      "logps/rejected": -28.338443756103516,
-      "loss": 1.9393,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.4530436396598816,
-      "rewards/margins": 0.4687787890434265,
-      "rewards/rejected": -0.015735208988189697,
+      "logits/chosen": -3.0124194622039795,
+      "logits/rejected": -2.997340679168701,
+      "logps/chosen": -29.7950382232666,
+      "logps/rejected": -28.343379974365234,
+      "loss": 1.1726,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.2082732617855072,
+      "rewards/margins": 0.22057661414146423,
+      "rewards/rejected": -0.012303322553634644,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 75.0,
+      "grad_norm": 101.0,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.931588649749756,
-      "logits/rejected": -2.9138498306274414,
-      "logps/chosen": -27.903301239013672,
-      "logps/rejected": -30.379735946655273,
-      "loss": 0.7854,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.36701759696006775,
-      "rewards/margins": 0.35749000310897827,
-      "rewards/rejected": 0.009527605026960373,
+      "logits/chosen": -2.9271466732025146,
+      "logits/rejected": -2.9091556072235107,
+      "logps/chosen": -27.729639053344727,
+      "logps/rejected": -30.502544403076172,
+      "loss": 1.2879,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.3162160813808441,
+      "rewards/margins": 0.3716646134853363,
+      "rewards/rejected": -0.05544847995042801,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 101.5,
+      "grad_norm": 154.0,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -3.1477503776550293,
-      "logits/rejected": -3.153796672821045,
-      "logps/chosen": -31.147018432617188,
-      "logps/rejected": -32.76397705078125,
-      "loss": 0.8634,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.39374879002571106,
-      "rewards/margins": 0.44812655448913574,
-      "rewards/rejected": -0.05437783524394035,
+      "logits/chosen": -3.141763687133789,
+      "logits/rejected": -3.147583246231079,
+      "logps/chosen": -31.143047332763672,
+      "logps/rejected": -32.79061508178711,
+      "loss": 1.3257,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.24807891249656677,
+      "rewards/margins": 0.2953856885433197,
+      "rewards/rejected": -0.04730679839849472,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 176.0,
+      "grad_norm": 67.5,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -3.023674249649048,
-      "logits/rejected": -3.0265414714813232,
-      "logps/chosen": -30.397380828857422,
-      "logps/rejected": -31.44447898864746,
-      "loss": 0.5848,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.3276014029979706,
-      "rewards/margins": 0.27253925800323486,
-      "rewards/rejected": 0.05506212264299393,
+      "logits/chosen": -3.018929958343506,
+      "logits/rejected": -3.0223464965820312,
+      "logps/chosen": -30.172698974609375,
+      "logps/rejected": -31.532190322875977,
+      "loss": 1.1227,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.3170902729034424,
+      "rewards/margins": 0.3265308439731598,
+      "rewards/rejected": -0.009440585970878601,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 262.0,
+      "grad_norm": 178.0,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -2.8648974895477295,
-      "logits/rejected": -2.866847515106201,
-      "logps/chosen": -28.0252628326416,
-      "logps/rejected": -29.79150390625,
-      "loss": 1.1652,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.49957290291786194,
-      "rewards/margins": 0.4680249094963074,
-      "rewards/rejected": 0.03154800459742546,
+      "logits/chosen": -2.860013246536255,
+      "logits/rejected": -2.861978769302368,
+      "logps/chosen": -27.91092872619629,
+      "logps/rejected": -29.804889678955078,
+      "loss": 1.038,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.3693995475769043,
+      "rewards/margins": 0.35637328028678894,
+      "rewards/rejected": 0.013026287779211998,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 161.0,
+      "grad_norm": 178.0,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.937594175338745,
-      "logits/rejected": -2.936284065246582,
-      "logps/chosen": -30.407318115234375,
-      "logps/rejected": -32.029598236083984,
-      "loss": 0.8323,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.30346399545669556,
-      "rewards/margins": 0.38470739126205444,
-      "rewards/rejected": -0.0812433734536171,
+      "logits/chosen": -2.9331061840057373,
+      "logits/rejected": -2.932417392730713,
+      "logps/chosen": -30.217334747314453,
+      "logps/rejected": -32.091407775878906,
+      "loss": 1.1083,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.28465867042541504,
+      "rewards/margins": 0.36634117364883423,
+      "rewards/rejected": -0.08168253302574158,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 520.0,
+      "grad_norm": 624.0,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -2.8965048789978027,
-      "logits/rejected": -2.879058837890625,
-      "logps/chosen": -28.52047348022461,
-      "logps/rejected": -27.761425018310547,
-      "loss": 0.7634,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.33964917063713074,
-      "rewards/margins": 0.23709365725517273,
-      "rewards/rejected": 0.10255551338195801,
+      "logits/chosen": -2.892216205596924,
+      "logits/rejected": -2.874706268310547,
+      "logps/chosen": -28.363414764404297,
+      "logps/rejected": -27.839080810546875,
+      "loss": 1.4701,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.29081061482429504,
+      "rewards/margins": 0.2655412554740906,
+      "rewards/rejected": 0.02526932954788208,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 1.1209261448352368,
-      "train_runtime": 2723.1709,
-      "train_samples_per_second": 1.131,
+      "train_loss": 1.4646434090354226,
+      "train_runtime": 2724.3946,
+      "train_samples_per_second": 1.13,
       "train_steps_per_second": 0.141
     }
   ],