diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,2958 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9997382884061764,
+  "eval_steps": 500,
+  "global_step": 1910,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0005234231876472127,
+      "grad_norm": 14.640187733663568,
+      "learning_rate": 2.617801047120419e-09,
+      "logits/chosen": -2.959083080291748,
+      "logits/rejected": -3.0469329357147217,
+      "logps/chosen": -134.99343872070312,
+      "logps/rejected": -161.30029296875,
+      "loss": 0.693,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": 0.0001932144077727571,
+      "rewards/margins": 0.0005058574606664479,
+      "rewards/rejected": -0.00031264303834177554,
+      "step": 1
+    },
+    {
+      "epoch": 0.005234231876472127,
+      "grad_norm": 14.254383428129056,
+      "learning_rate": 2.6178010471204188e-08,
+      "logits/chosen": -2.9816136360168457,
+      "logits/rejected": -2.893190860748291,
+      "logps/chosen": -186.0762176513672,
+      "logps/rejected": -186.07012939453125,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4583333432674408,
+      "rewards/chosen": 0.0004193554923404008,
+      "rewards/margins": -0.00021269405260682106,
+      "rewards/rejected": 0.0006320495158433914,
+      "step": 10
+    },
+    {
+      "epoch": 0.010468463752944255,
+      "grad_norm": 12.949573017221987,
+      "learning_rate": 5.2356020942408376e-08,
+      "logits/chosen": -2.829813003540039,
+      "logits/rejected": -2.8478147983551025,
+      "logps/chosen": -174.7886199951172,
+      "logps/rejected": -193.36080932617188,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.0009858906269073486,
+      "rewards/margins": -0.0011966052697971463,
+      "rewards/rejected": 0.00021071455557830632,
+      "step": 20
+    },
+    {
+      "epoch": 0.015702695629416383,
+      "grad_norm": 15.371844314146115,
+      "learning_rate": 7.853403141361257e-08,
+      "logits/chosen": -3.01656174659729,
+      "logits/rejected": -2.9938042163848877,
+      "logps/chosen": -214.79541015625,
+      "logps/rejected": -201.54867553710938,
+      "loss": 0.693,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.000996897229924798,
+      "rewards/margins": 0.00021716937771998346,
+      "rewards/rejected": -0.00121406652033329,
+      "step": 30
+    },
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 14.22366978101997,
+      "learning_rate": 1.0471204188481675e-07,
+      "logits/chosen": -2.874187469482422,
+      "logits/rejected": -2.886640787124634,
+      "logps/chosen": -172.05311584472656,
+      "logps/rejected": -216.9876251220703,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.002353773918002844,
+      "rewards/margins": 0.0017049995949491858,
+      "rewards/rejected": -0.004058773163706064,
+      "step": 40
+    },
+    {
+      "epoch": 0.02617115938236064,
+      "grad_norm": 13.356014407693477,
+      "learning_rate": 1.3089005235602092e-07,
+      "logits/chosen": -2.9540908336639404,
+      "logits/rejected": -2.9672648906707764,
+      "logps/chosen": -196.1494598388672,
+      "logps/rejected": -180.84803771972656,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.00872573908418417,
+      "rewards/margins": 0.002879455918446183,
+      "rewards/rejected": -0.01160519476979971,
+      "step": 50
+    },
+    {
+      "epoch": 0.031405391258832765,
+      "grad_norm": 14.564330446769505,
+      "learning_rate": 1.5706806282722514e-07,
+      "logits/chosen": -2.9785428047180176,
+      "logits/rejected": -2.9725818634033203,
+      "logps/chosen": -183.6147918701172,
+      "logps/rejected": -203.66824340820312,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.01978887803852558,
+      "rewards/margins": 0.005196017678827047,
+      "rewards/rejected": -0.024984892457723618,
+      "step": 60
+    },
+    {
+      "epoch": 0.036639623135304895,
+      "grad_norm": 16.449869342928505,
+      "learning_rate": 1.8324607329842932e-07,
+      "logits/chosen": -2.995837688446045,
+      "logits/rejected": -2.8539786338806152,
+      "logps/chosen": -197.07168579101562,
+      "logps/rejected": -206.65469360351562,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.02455015294253826,
+      "rewards/margins": 0.011808360926806927,
+      "rewards/rejected": -0.03635851666331291,
+      "step": 70
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 14.79926563160046,
+      "learning_rate": 2.094240837696335e-07,
+      "logits/chosen": -2.9914395809173584,
+      "logits/rejected": -2.942915439605713,
+      "logps/chosen": -205.7972869873047,
+      "logps/rejected": -192.72232055664062,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.04919552057981491,
+      "rewards/margins": 0.009290899150073528,
+      "rewards/rejected": -0.058486420661211014,
+      "step": 80
+    },
+    {
+      "epoch": 0.04710808688824915,
+      "grad_norm": 14.05004280238572,
+      "learning_rate": 2.356020942408377e-07,
+      "logits/chosen": -2.888996124267578,
+      "logits/rejected": -2.804115056991577,
+      "logps/chosen": -184.47732543945312,
+      "logps/rejected": -172.6226348876953,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07440824806690216,
+      "rewards/margins": 0.013659747317433357,
+      "rewards/rejected": -0.08806798607110977,
+      "step": 90
+    },
+    {
+      "epoch": 0.05234231876472128,
+      "grad_norm": 13.559496262792273,
+      "learning_rate": 2.6178010471204185e-07,
+      "logits/chosen": -2.9032299518585205,
+      "logits/rejected": -2.857269525527954,
+      "logps/chosen": -206.85140991210938,
+      "logps/rejected": -219.37026977539062,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0838988870382309,
+      "rewards/margins": 0.028635507449507713,
+      "rewards/rejected": -0.11253438144922256,
+      "step": 100
+    },
+    {
+      "epoch": 0.05757655064119341,
+      "grad_norm": 13.4996248822944,
+      "learning_rate": 2.879581151832461e-07,
+      "logits/chosen": -2.940826892852783,
+      "logits/rejected": -2.8834738731384277,
+      "logps/chosen": -212.55807495117188,
+      "logps/rejected": -201.2547607421875,
+      "loss": 0.675,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.10653187334537506,
+      "rewards/margins": 0.05816306918859482,
+      "rewards/rejected": -0.16469493508338928,
+      "step": 110
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 12.884354544368191,
+      "learning_rate": 3.1413612565445027e-07,
+      "logits/chosen": -2.927412986755371,
+      "logits/rejected": -2.985417604446411,
+      "logps/chosen": -228.93350219726562,
+      "logps/rejected": -238.04269409179688,
+      "loss": 0.6747,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.10475633293390274,
+      "rewards/margins": 0.046889565885066986,
+      "rewards/rejected": -0.15164589881896973,
+      "step": 120
+    },
+    {
+      "epoch": 0.06804501439413765,
+      "grad_norm": 14.312004568183767,
+      "learning_rate": 3.4031413612565446e-07,
+      "logits/chosen": -2.8365414142608643,
+      "logits/rejected": -2.876112461090088,
+      "logps/chosen": -200.61264038085938,
+      "logps/rejected": -213.1061248779297,
+      "loss": 0.6622,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.13382001221179962,
+      "rewards/margins": 0.06009231135249138,
+      "rewards/rejected": -0.1939123272895813,
+      "step": 130
+    },
+    {
+      "epoch": 0.07327924627060979,
+      "grad_norm": 15.573435236138954,
+      "learning_rate": 3.6649214659685864e-07,
+      "logits/chosen": -2.8712472915649414,
+      "logits/rejected": -2.8083159923553467,
+      "logps/chosen": -212.3485870361328,
+      "logps/rejected": -216.5910186767578,
+      "loss": 0.659,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2070785015821457,
+      "rewards/margins": 0.06957153975963593,
+      "rewards/rejected": -0.27665001153945923,
+      "step": 140
+    },
+    {
+      "epoch": 0.07851347814708191,
+      "grad_norm": 17.962582071011855,
+      "learning_rate": 3.926701570680628e-07,
+      "logits/chosen": -2.8894333839416504,
+      "logits/rejected": -2.783268928527832,
+      "logps/chosen": -211.5115509033203,
+      "logps/rejected": -224.9191131591797,
+      "loss": 0.6492,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.206833153963089,
+      "rewards/margins": 0.1691361963748932,
+      "rewards/rejected": -0.3759693503379822,
+      "step": 150
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 17.592745804055944,
+      "learning_rate": 4.18848167539267e-07,
+      "logits/chosen": -2.866983413696289,
+      "logits/rejected": -2.766951084136963,
+      "logps/chosen": -217.34207153320312,
+      "logps/rejected": -228.534912109375,
+      "loss": 0.6384,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.25625622272491455,
+      "rewards/margins": 0.15949755907058716,
+      "rewards/rejected": -0.4157537817955017,
+      "step": 160
+    },
+    {
+      "epoch": 0.08898194190002617,
+      "grad_norm": 17.652878273699034,
+      "learning_rate": 4.450261780104712e-07,
+      "logits/chosen": -2.8391916751861572,
+      "logits/rejected": -2.832740306854248,
+      "logps/chosen": -199.69781494140625,
+      "logps/rejected": -227.84652709960938,
+      "loss": 0.6301,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.13288532197475433,
+      "rewards/margins": 0.19410356879234314,
+      "rewards/rejected": -0.32698890566825867,
+      "step": 170
+    },
+    {
+      "epoch": 0.0942161737764983,
+      "grad_norm": 17.777586322420262,
+      "learning_rate": 4.712041884816754e-07,
+      "logits/chosen": -2.715904951095581,
+      "logits/rejected": -2.5992846488952637,
+      "logps/chosen": -210.23941040039062,
+      "logps/rejected": -229.0234832763672,
+      "loss": 0.6307,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.21204690635204315,
+      "rewards/margins": 0.17581574618816376,
+      "rewards/rejected": -0.3878627121448517,
+      "step": 180
+    },
+    {
+      "epoch": 0.09945040565297043,
+      "grad_norm": 28.072972397122147,
+      "learning_rate": 4.973821989528796e-07,
+      "logits/chosen": -2.6811397075653076,
+      "logits/rejected": -2.6616740226745605,
+      "logps/chosen": -216.45114135742188,
+      "logps/rejected": -258.03021240234375,
+      "loss": 0.6089,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.33117493987083435,
+      "rewards/margins": 0.33262476325035095,
+      "rewards/rejected": -0.6637996435165405,
+      "step": 190
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 29.910313896117426,
+      "learning_rate": 4.999661831436498e-07,
+      "logits/chosen": -2.5953736305236816,
+      "logits/rejected": -2.6287741661071777,
+      "logps/chosen": -201.4033966064453,
+      "logps/rejected": -260.02728271484375,
+      "loss": 0.6132,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.2563866972923279,
+      "rewards/margins": 0.3042460083961487,
+      "rewards/rejected": -0.5606327652931213,
+      "step": 200
+    },
+    {
+      "epoch": 0.10991886940591468,
+      "grad_norm": 35.27450975604578,
+      "learning_rate": 4.998492971140339e-07,
+      "logits/chosen": -2.543915271759033,
+      "logits/rejected": -2.3116703033447266,
+      "logps/chosen": -247.22891235351562,
+      "logps/rejected": -270.3451843261719,
+      "loss": 0.6417,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.595511794090271,
+      "rewards/margins": 0.2611444592475891,
+      "rewards/rejected": -0.8566562533378601,
+      "step": 210
+    },
+    {
+      "epoch": 0.11515310128238682,
+      "grad_norm": 48.58365341922755,
+      "learning_rate": 4.996489634487865e-07,
+      "logits/chosen": -2.545841932296753,
+      "logits/rejected": -2.2179739475250244,
+      "logps/chosen": -234.15786743164062,
+      "logps/rejected": -287.24920654296875,
+      "loss": 0.585,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.533110499382019,
+      "rewards/margins": 0.5109399557113647,
+      "rewards/rejected": -1.0440504550933838,
+      "step": 220
+    },
+    {
+      "epoch": 0.12038733315885894,
+      "grad_norm": 75.83009735431122,
+      "learning_rate": 4.993652490577246e-07,
+      "logits/chosen": -2.5557217597961426,
+      "logits/rejected": -2.3648390769958496,
+      "logps/chosen": -249.6676483154297,
+      "logps/rejected": -277.823486328125,
+      "loss": 0.6025,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5915195941925049,
+      "rewards/margins": 0.36318284273147583,
+      "rewards/rejected": -0.9547024965286255,
+      "step": 230
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 33.52383186890237,
+      "learning_rate": 4.9899824869915e-07,
+      "logits/chosen": -2.333035945892334,
+      "logits/rejected": -2.079817295074463,
+      "logps/chosen": -255.44595336914062,
+      "logps/rejected": -313.8713684082031,
+      "loss": 0.5681,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.8247841000556946,
+      "rewards/margins": 0.47075214982032776,
+      "rewards/rejected": -1.2955362796783447,
+      "step": 240
+    },
+    {
+      "epoch": 0.13085579691180318,
+      "grad_norm": 35.13466621709411,
+      "learning_rate": 4.985480849482012e-07,
+      "logits/chosen": -2.3787760734558105,
+      "logits/rejected": -2.364150047302246,
+      "logps/chosen": -304.33441162109375,
+      "logps/rejected": -327.4510498046875,
+      "loss": 0.5645,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.0099525451660156,
+      "rewards/margins": 0.29796892404556274,
+      "rewards/rejected": -1.3079214096069336,
+      "step": 250
+    },
+    {
+      "epoch": 0.1360900287882753,
+      "grad_norm": 33.359619674605376,
+      "learning_rate": 4.980149081559142e-07,
+      "logits/chosen": -2.2392945289611816,
+      "logits/rejected": -2.0072715282440186,
+      "logps/chosen": -299.57879638671875,
+      "logps/rejected": -395.2071838378906,
+      "loss": 0.5998,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.3633458614349365,
+      "rewards/margins": 0.728509247303009,
+      "rewards/rejected": -2.09185528755188,
+      "step": 260
+    },
+    {
+      "epoch": 0.14132426066474746,
+      "grad_norm": 47.681686539836825,
+      "learning_rate": 4.973988963990065e-07,
+      "logits/chosen": -2.056598424911499,
+      "logits/rejected": -2.0986738204956055,
+      "logps/chosen": -318.61956787109375,
+      "logps/rejected": -412.1875915527344,
+      "loss": 0.548,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.372604489326477,
+      "rewards/margins": 0.7224138975143433,
+      "rewards/rejected": -2.095018148422241,
+      "step": 270
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 30.114115223768707,
+      "learning_rate": 4.967002554204008e-07,
+      "logits/chosen": -2.0974724292755127,
+      "logits/rejected": -2.0098748207092285,
+      "logps/chosen": -350.09600830078125,
+      "logps/rejected": -403.1016540527344,
+      "loss": 0.5879,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6619269847869873,
+      "rewards/margins": 0.39111390709877014,
+      "rewards/rejected": -2.0530409812927246,
+      "step": 280
+    },
+    {
+      "epoch": 0.1517927244176917,
+      "grad_norm": 31.769810828425065,
+      "learning_rate": 4.959192185605087e-07,
+      "logits/chosen": -2.51796817779541,
+      "logits/rejected": -2.2334158420562744,
+      "logps/chosen": -324.0982360839844,
+      "logps/rejected": -340.4048767089844,
+      "loss": 0.5945,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1041514873504639,
+      "rewards/margins": 0.38266053795814514,
+      "rewards/rejected": -1.486811876296997,
+      "step": 290
+    },
+    {
+      "epoch": 0.15702695629416383,
+      "grad_norm": 48.774597859411024,
+      "learning_rate": 4.950560466792969e-07,
+      "logits/chosen": -2.362421751022339,
+      "logits/rejected": -2.193986654281616,
+      "logps/chosen": -305.4976806640625,
+      "logps/rejected": -367.10467529296875,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.2003720998764038,
+      "rewards/margins": 0.572384774684906,
+      "rewards/rejected": -1.7727569341659546,
+      "step": 300
+    },
+    {
+      "epoch": 0.16226118817063595,
+      "grad_norm": 39.19160116710573,
+      "learning_rate": 4.941110280691619e-07,
+      "logits/chosen": -2.353766441345215,
+      "logits/rejected": -2.0096635818481445,
+      "logps/chosen": -309.58905029296875,
+      "logps/rejected": -353.7257080078125,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.201560378074646,
+      "rewards/margins": 0.48785895109176636,
+      "rewards/rejected": -1.6894193887710571,
+      "step": 310
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 37.587189222436905,
+      "learning_rate": 4.930844783586424e-07,
+      "logits/chosen": -2.396035671234131,
+      "logits/rejected": -2.165098190307617,
+      "logps/chosen": -288.3567810058594,
+      "logps/rejected": -332.8038635253906,
+      "loss": 0.5415,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.062434196472168,
+      "rewards/margins": 0.4423864781856537,
+      "rewards/rejected": -1.5048205852508545,
+      "step": 320
+    },
+    {
+      "epoch": 0.17272965192358022,
+      "grad_norm": 50.8199153591931,
+      "learning_rate": 4.919767404070033e-07,
+      "logits/chosen": -1.8470354080200195,
+      "logits/rejected": -1.600237250328064,
+      "logps/chosen": -307.25360107421875,
+      "logps/rejected": -398.39910888671875,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3850140571594238,
+      "rewards/margins": 0.7685943245887756,
+      "rewards/rejected": -2.153608560562134,
+      "step": 330
+    },
+    {
+      "epoch": 0.17796388380005235,
+      "grad_norm": 39.2860888155997,
+      "learning_rate": 4.907881841897216e-07,
+      "logits/chosen": -2.0841307640075684,
+      "logits/rejected": -1.8823187351226807,
+      "logps/chosen": -306.69305419921875,
+      "logps/rejected": -375.80352783203125,
+      "loss": 0.5377,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.2261284589767456,
+      "rewards/margins": 0.7133340835571289,
+      "rewards/rejected": -1.939462423324585,
+      "step": 340
+    },
+    {
+      "epoch": 0.18319811567652447,
+      "grad_norm": 44.441009515641795,
+      "learning_rate": 4.895192066749189e-07,
+      "logits/chosen": -2.40903902053833,
+      "logits/rejected": -2.2892374992370605,
+      "logps/chosen": -299.1591796875,
+      "logps/rejected": -401.603271484375,
+      "loss": 0.5481,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1807595491409302,
+      "rewards/margins": 0.7055211663246155,
+      "rewards/rejected": -1.8862807750701904,
+      "step": 350
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 33.989270442301404,
+      "learning_rate": 4.881702316907768e-07,
+      "logits/chosen": -2.2026100158691406,
+      "logits/rejected": -2.008274793624878,
+      "logps/chosen": -290.2915344238281,
+      "logps/rejected": -346.9622497558594,
+      "loss": 0.5596,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1741074323654175,
+      "rewards/margins": 0.5104838609695435,
+      "rewards/rejected": -1.68459153175354,
+      "step": 360
+    },
+    {
+      "epoch": 0.19366657942946872,
+      "grad_norm": 39.74252231849878,
+      "learning_rate": 4.86741709783982e-07,
+      "logits/chosen": -2.0070841312408447,
+      "logits/rejected": -1.5279141664505005,
+      "logps/chosen": -293.36578369140625,
+      "logps/rejected": -345.587646484375,
+      "loss": 0.5468,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2320070266723633,
+      "rewards/margins": 0.6169141530990601,
+      "rewards/rejected": -1.8489214181900024,
+      "step": 370
+    },
+    {
+      "epoch": 0.19890081130594087,
+      "grad_norm": 41.66915169536549,
+      "learning_rate": 4.85234118069247e-07,
+      "logits/chosen": -1.4260574579238892,
+      "logits/rejected": -1.1182783842086792,
+      "logps/chosen": -323.9841003417969,
+      "logps/rejected": -404.05517578125,
+      "loss": 0.5282,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.595336675643921,
+      "rewards/margins": 0.6996889114379883,
+      "rewards/rejected": -2.295025587081909,
+      "step": 380
+    },
+    {
+      "epoch": 0.204135043182413,
+      "grad_norm": 38.40542199265962,
+      "learning_rate": 4.836479600699578e-07,
+      "logits/chosen": -1.7142446041107178,
+      "logits/rejected": -1.7196638584136963,
+      "logps/chosen": -361.3770446777344,
+      "logps/rejected": -439.6658630371094,
+      "loss": 0.5487,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.594753623008728,
+      "rewards/margins": 0.683542788028717,
+      "rewards/rejected": -2.27829647064209,
+      "step": 390
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 39.63693507119565,
+      "learning_rate": 4.819837655500013e-07,
+      "logits/chosen": -1.911871314048767,
+      "logits/rejected": -1.7180120944976807,
+      "logps/chosen": -339.29705810546875,
+      "logps/rejected": -399.7021179199219,
+      "loss": 0.54,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.528294563293457,
+      "rewards/margins": 0.6349833011627197,
+      "rewards/rejected": -2.163278102874756,
+      "step": 400
+    },
+    {
+      "epoch": 0.21460350693535724,
+      "grad_norm": 39.95185592857185,
+      "learning_rate": 4.802420903368285e-07,
+      "logits/chosen": -1.7220875024795532,
+      "logits/rejected": -1.3575185537338257,
+      "logps/chosen": -365.69964599609375,
+      "logps/rejected": -418.9811096191406,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7411553859710693,
+      "rewards/margins": 0.48729467391967773,
+      "rewards/rejected": -2.228450059890747,
+      "step": 410
+    },
+    {
+      "epoch": 0.21983773881182936,
+      "grad_norm": 42.46558644572104,
+      "learning_rate": 4.784235161358123e-07,
+      "logits/chosen": -1.8128083944320679,
+      "logits/rejected": -1.8185402154922485,
+      "logps/chosen": -309.54937744140625,
+      "logps/rejected": -385.9242248535156,
+      "loss": 0.5788,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.3886371850967407,
+      "rewards/margins": 0.5779408812522888,
+      "rewards/rejected": -1.9665781259536743,
+      "step": 420
+    },
+    {
+      "epoch": 0.22507197068830148,
+      "grad_norm": 40.391730044431704,
+      "learning_rate": 4.7652865033596314e-07,
+      "logits/chosen": -2.0464603900909424,
+      "logits/rejected": -1.5721585750579834,
+      "logps/chosen": -317.18560791015625,
+      "logps/rejected": -354.8387756347656,
+      "loss": 0.5339,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.305790662765503,
+      "rewards/margins": 0.5792006254196167,
+      "rewards/rejected": -1.8849912881851196,
+      "step": 430
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 44.15647862665091,
+      "learning_rate": 4.7455812580706534e-07,
+      "logits/chosen": -2.0872130393981934,
+      "logits/rejected": -1.5626317262649536,
+      "logps/chosen": -345.29278564453125,
+      "logps/rejected": -409.45989990234375,
+      "loss": 0.549,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5832579135894775,
+      "rewards/margins": 0.6660935282707214,
+      "rewards/rejected": -2.2493512630462646,
+      "step": 440
+    },
+    {
+      "epoch": 0.23554043444124576,
+      "grad_norm": 42.04086790745509,
+      "learning_rate": 4.725126006883046e-07,
+      "logits/chosen": -1.773993730545044,
+      "logits/rejected": -1.5137646198272705,
+      "logps/chosen": -326.9532165527344,
+      "logps/rejected": -401.57867431640625,
+      "loss": 0.577,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5454232692718506,
+      "rewards/margins": 0.6505107879638672,
+      "rewards/rejected": -2.1959340572357178,
+      "step": 450
+    },
+    {
+      "epoch": 0.24077466631771788,
+      "grad_norm": 36.31044897637686,
+      "learning_rate": 4.703927581684539e-07,
+      "logits/chosen": -2.1727023124694824,
+      "logits/rejected": -1.8378498554229736,
+      "logps/chosen": -350.1428527832031,
+      "logps/rejected": -373.5480041503906,
+      "loss": 0.5687,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4619617462158203,
+      "rewards/margins": 0.49050265550613403,
+      "rewards/rejected": -1.9524643421173096,
+      "step": 460
+    },
+    {
+      "epoch": 0.24600889819419,
+      "grad_norm": 38.97906947550607,
+      "learning_rate": 4.68199306257695e-07,
+      "logits/chosen": -1.960343599319458,
+      "logits/rejected": -1.3587312698364258,
+      "logps/chosen": -331.9888916015625,
+      "logps/rejected": -399.2967834472656,
+      "loss": 0.4892,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4141899347305298,
+      "rewards/margins": 0.7412781715393066,
+      "rewards/rejected": -2.155468225479126,
+      "step": 470
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 60.96840256666392,
+      "learning_rate": 4.6593297755114776e-07,
+      "logits/chosen": -1.4715737104415894,
+      "logits/rejected": -0.8570035099983215,
+      "logps/chosen": -363.36376953125,
+      "logps/rejected": -419.7437438964844,
+      "loss": 0.5466,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9223047494888306,
+      "rewards/margins": 0.5888181328773499,
+      "rewards/rejected": -2.511122941970825,
+      "step": 480
+    },
+    {
+      "epoch": 0.2564773619471343,
+      "grad_norm": 56.85832650681642,
+      "learning_rate": 4.635945289841902e-07,
+      "logits/chosen": -1.736699104309082,
+      "logits/rejected": -1.6159862279891968,
+      "logps/chosen": -331.16339111328125,
+      "logps/rejected": -429.3055114746094,
+      "loss": 0.5324,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.511039137840271,
+      "rewards/margins": 0.6905651092529297,
+      "rewards/rejected": -2.2016043663024902,
+      "step": 490
+    },
+    {
+      "epoch": 0.26171159382360637,
+      "grad_norm": 45.91788216290567,
+      "learning_rate": 4.611847415796476e-07,
+      "logits/chosen": -1.6824716329574585,
+      "logits/rejected": -1.536476731300354,
+      "logps/chosen": -365.19488525390625,
+      "logps/rejected": -471.90985107421875,
+      "loss": 0.5294,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.727085828781128,
+      "rewards/margins": 0.8007200956344604,
+      "rewards/rejected": -2.527806043624878,
+      "step": 500
+    },
+    {
+      "epoch": 0.26171159382360637,
+      "eval_logits/chosen": -0.863903820514679,
+      "eval_logits/rejected": -0.4723878502845764,
+      "eval_logps/chosen": -358.6011047363281,
+      "eval_logps/rejected": -428.4360656738281,
+      "eval_loss": 0.5469957590103149,
+      "eval_rewards/accuracies": 0.6980000138282776,
+      "eval_rewards/chosen": -1.7358005046844482,
+      "eval_rewards/margins": 0.6566510796546936,
+      "eval_rewards/rejected": -2.392451763153076,
+      "eval_runtime": 316.8135,
+      "eval_samples_per_second": 6.313,
+      "eval_steps_per_second": 0.789,
+      "step": 500
+    },
+    {
+      "epoch": 0.2669458257000785,
+      "grad_norm": 37.615935219623054,
+      "learning_rate": 4.5870442018693773e-07,
+      "logits/chosen": -1.8111470937728882,
+      "logits/rejected": -1.5349056720733643,
+      "logps/chosen": -327.2267761230469,
+      "logps/rejected": -381.962890625,
+      "loss": 0.5606,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.547119140625,
+      "rewards/margins": 0.5296986699104309,
+      "rewards/rejected": -2.076817750930786,
+      "step": 510
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 35.85517663702724,
+      "learning_rate": 4.5615439321325735e-07,
+      "logits/chosen": -2.0327141284942627,
+      "logits/rejected": -1.460468053817749,
+      "logps/chosen": -312.06671142578125,
+      "logps/rejected": -395.5126647949219,
+      "loss": 0.5241,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.3687623739242554,
+      "rewards/margins": 0.6768960952758789,
+      "rewards/rejected": -2.045658588409424,
+      "step": 520
+    },
+    {
+      "epoch": 0.27741428945302277,
+      "grad_norm": 50.37074560251419,
+      "learning_rate": 4.535355123469008e-07,
+      "logits/chosen": -1.4753310680389404,
+      "logits/rejected": -1.471920132637024,
+      "logps/chosen": -338.68060302734375,
+      "logps/rejected": -424.1499938964844,
+      "loss": 0.545,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7495324611663818,
+      "rewards/margins": 0.5848132371902466,
+      "rewards/rejected": -2.334345579147339,
+      "step": 530
+    },
+    {
+      "epoch": 0.2826485213294949,
+      "grad_norm": 55.36008562841294,
+      "learning_rate": 4.5084865227280366e-07,
+      "logits/chosen": -1.087989091873169,
+      "logits/rejected": -1.2594499588012695,
+      "logps/chosen": -424.70941162109375,
+      "logps/rejected": -481.74688720703125,
+      "loss": 0.5205,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.1392130851745605,
+      "rewards/margins": 0.6303030252456665,
+      "rewards/rejected": -2.7695162296295166,
+      "step": 540
+    },
+    {
+      "epoch": 0.287882753205967,
+      "grad_norm": 39.75849189232047,
+      "learning_rate": 4.4809471038040437e-07,
+      "logits/chosen": -1.642165184020996,
+      "logits/rejected": -0.9477740526199341,
+      "logps/chosen": -399.1435546875,
+      "logps/rejected": -444.47882080078125,
+      "loss": 0.5659,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8148092031478882,
+      "rewards/margins": 0.7540385127067566,
+      "rewards/rejected": -2.568847894668579,
+      "step": 550
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 33.49141951717591,
+      "learning_rate": 4.4527460646392386e-07,
+      "logits/chosen": -2.0694565773010254,
+      "logits/rejected": -1.7511459589004517,
+      "logps/chosen": -310.82574462890625,
+      "logps/rejected": -369.46539306640625,
+      "loss": 0.5329,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1495193243026733,
+      "rewards/margins": 0.6521326303482056,
+      "rewards/rejected": -1.801652193069458,
+      "step": 560
+    },
+    {
+      "epoch": 0.29835121695891126,
+      "grad_norm": 56.88589774049656,
+      "learning_rate": 4.4238928241516163e-07,
+      "logits/chosen": -1.8898957967758179,
+      "logits/rejected": -1.2891989946365356,
+      "logps/chosen": -350.3455810546875,
+      "logps/rejected": -402.1921081542969,
+      "loss": 0.5373,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.413482427597046,
+      "rewards/margins": 0.6170186996459961,
+      "rewards/rejected": -2.030501127243042,
+      "step": 570
+    },
+    {
+      "epoch": 0.3035854488353834,
+      "grad_norm": 42.10520204001843,
+      "learning_rate": 4.394397019089116e-07,
+      "logits/chosen": -1.6958551406860352,
+      "logits/rejected": -0.6804537773132324,
+      "logps/chosen": -387.3909606933594,
+      "logps/rejected": -451.7494201660156,
+      "loss": 0.544,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.97348952293396,
+      "rewards/margins": 0.7865778207778931,
+      "rewards/rejected": -2.7600669860839844,
+      "step": 580
+    },
+    {
+      "epoch": 0.30881968071185556,
+      "grad_norm": 35.07476918232387,
+      "learning_rate": 4.3642685008110246e-07,
+      "logits/chosen": -1.4044368267059326,
+      "logits/rejected": -0.7555860280990601,
+      "logps/chosen": -428.0267639160156,
+      "logps/rejected": -496.0921936035156,
+      "loss": 0.5421,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.2372324466705322,
+      "rewards/margins": 0.8896536827087402,
+      "rewards/rejected": -3.1268858909606934,
+      "step": 590
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 56.64712231153677,
+      "learning_rate": 4.333517331997704e-07,
+      "logits/chosen": -1.425999641418457,
+      "logits/rejected": -0.7339752912521362,
+      "logps/chosen": -357.58050537109375,
+      "logps/rejected": -459.1241149902344,
+      "loss": 0.5675,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.811273217201233,
+      "rewards/margins": 0.9072386622428894,
+      "rewards/rejected": -2.7185120582580566,
+      "step": 600
+    },
+    {
+      "epoch": 0.3192881444647998,
+      "grad_norm": 53.88796332939181,
+      "learning_rate": 4.302153783289736e-07,
+      "logits/chosen": -1.348676085472107,
+      "logits/rejected": -0.9413366317749023,
+      "logps/chosen": -354.7875061035156,
+      "logps/rejected": -408.4991149902344,
+      "loss": 0.5262,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5594650506973267,
+      "rewards/margins": 0.6488285064697266,
+      "rewards/rejected": -2.2082934379577637,
+      "step": 610
+    },
+    {
+      "epoch": 0.3245223763412719,
+      "grad_norm": 52.50744960152232,
+      "learning_rate": 4.2701883298576124e-07,
+      "logits/chosen": -1.720445990562439,
+      "logits/rejected": -1.290917992591858,
+      "logps/chosen": -375.7829284667969,
+      "logps/rejected": -434.217529296875,
+      "loss": 0.5917,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6508047580718994,
+      "rewards/margins": 0.5466897487640381,
+      "rewards/rejected": -2.1974947452545166,
+      "step": 620
+    },
+    {
+      "epoch": 0.32975660821774405,
+      "grad_norm": 44.222624438240096,
+      "learning_rate": 4.237631647903115e-07,
+      "logits/chosen": -1.4673423767089844,
+      "logits/rejected": -0.8496716618537903,
+      "logps/chosen": -377.01470947265625,
+      "logps/rejected": -444.0126953125,
+      "loss": 0.5623,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.9182418584823608,
+      "rewards/margins": 0.6217330694198608,
+      "rewards/rejected": -2.539975166320801,
+      "step": 630
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 56.74613947557357,
+      "learning_rate": 4.204494611093548e-07,
+      "logits/chosen": -1.0175368785858154,
+      "logits/rejected": -1.0307013988494873,
+      "logps/chosen": -352.26666259765625,
+      "logps/rejected": -444.2943420410156,
+      "loss": 0.5419,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.003497362136841,
+      "rewards/margins": 0.6112686395645142,
+      "rewards/rejected": -2.6147658824920654,
+      "step": 640
+    },
+    {
+      "epoch": 0.3402250719706883,
+      "grad_norm": 56.35474765839724,
+      "learning_rate": 4.1707882869300235e-07,
+      "logits/chosen": -1.4247758388519287,
+      "logits/rejected": -0.8727043271064758,
+      "logps/chosen": -357.51800537109375,
+      "logps/rejected": -452.87677001953125,
+      "loss": 0.526,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.8759056329727173,
+      "rewards/margins": 0.9082478284835815,
+      "rewards/rejected": -2.784153699874878,
+      "step": 650
+    },
+    {
+      "epoch": 0.34545930384716045,
+      "grad_norm": 51.29897875137078,
+      "learning_rate": 4.136523933051005e-07,
+      "logits/chosen": -1.3449084758758545,
+      "logits/rejected": -1.0442228317260742,
+      "logps/chosen": -372.7139587402344,
+      "logps/rejected": -517.5125732421875,
+      "loss": 0.5216,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0670166015625,
+      "rewards/margins": 1.0036545991897583,
+      "rewards/rejected": -3.0706706047058105,
+      "step": 660
+    },
+    {
+      "epoch": 0.35069353572363254,
+      "grad_norm": 53.649661826724454,
+      "learning_rate": 4.101712993472348e-07,
+      "logits/chosen": -1.49883234500885,
+      "logits/rejected": -1.0886845588684082,
+      "logps/chosen": -370.93817138671875,
+      "logps/rejected": -424.97064208984375,
+      "loss": 0.5716,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9939110279083252,
+      "rewards/margins": 0.5497623085975647,
+      "rewards/rejected": -2.543673276901245,
+      "step": 670
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 44.18426608864023,
+      "learning_rate": 4.066367094765091e-07,
+      "logits/chosen": -1.3670942783355713,
+      "logits/rejected": -1.2187774181365967,
+      "logps/chosen": -374.8455810546875,
+      "logps/rejected": -434.95867919921875,
+      "loss": 0.535,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9276262521743774,
+      "rewards/margins": 0.5358133912086487,
+      "rewards/rejected": -2.463439464569092,
+      "step": 680
+    },
+    {
+      "epoch": 0.3611619994765768,
+      "grad_norm": 52.799712890225976,
+      "learning_rate": 4.0304980421722766e-07,
+      "logits/chosen": -1.5443974733352661,
+      "logits/rejected": -0.7440096139907837,
+      "logps/chosen": -365.8990173339844,
+      "logps/rejected": -443.71368408203125,
+      "loss": 0.5022,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7589327096939087,
+      "rewards/margins": 0.8254102468490601,
+      "rewards/rejected": -2.5843429565429688,
+      "step": 690
+    },
+    {
+      "epoch": 0.36639623135304894,
+      "grad_norm": 51.1426156745725,
+      "learning_rate": 3.994117815666095e-07,
+      "logits/chosen": -1.7846711874008179,
+      "logits/rejected": -0.7315829992294312,
+      "logps/chosen": -387.86376953125,
+      "logps/rejected": -462.34503173828125,
+      "loss": 0.5186,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9715665578842163,
+      "rewards/margins": 0.752856969833374,
+      "rewards/rejected": -2.724423885345459,
+      "step": 700
+    },
+    {
+      "epoch": 0.3716304632295211,
+      "grad_norm": 62.36681697005891,
+      "learning_rate": 3.957238565946671e-07,
+      "logits/chosen": -1.2870978116989136,
+      "logits/rejected": -0.7070925235748291,
+      "logps/chosen": -403.86444091796875,
+      "logps/rejected": -515.5906982421875,
+      "loss": 0.483,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.2011804580688477,
+      "rewards/margins": 1.022341012954712,
+      "rewards/rejected": -3.2235217094421387,
+      "step": 710
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 57.328832948777865,
+      "learning_rate": 3.9198726103838306e-07,
+      "logits/chosen": -1.2161122560501099,
+      "logits/rejected": -0.788101315498352,
+      "logps/chosen": -387.68768310546875,
+      "logps/rejected": -519.7088012695312,
+      "loss": 0.5069,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9085991382598877,
+      "rewards/margins": 1.1666193008422852,
+      "rewards/rejected": -3.0752182006835938,
+      "step": 720
+    },
+    {
+      "epoch": 0.38209892698246534,
+      "grad_norm": 68.04530741097648,
+      "learning_rate": 3.8820324289031946e-07,
+      "logits/chosen": -1.356323003768921,
+      "logits/rejected": -1.176628828048706,
+      "logps/chosen": -346.1966247558594,
+      "logps/rejected": -447.4593811035156,
+      "loss": 0.5455,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.8683630228042603,
+      "rewards/margins": 0.8649228811264038,
+      "rewards/rejected": -2.733285903930664,
+      "step": 730
+    },
+    {
+      "epoch": 0.38733315885893743,
+      "grad_norm": 39.4868442275961,
+      "learning_rate": 3.84373065981799e-07,
+      "logits/chosen": -1.5190155506134033,
+      "logits/rejected": -1.4792649745941162,
+      "logps/chosen": -346.9692077636719,
+      "logps/rejected": -456.06634521484375,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.7277275323867798,
+      "rewards/margins": 0.8931981921195984,
+      "rewards/rejected": -2.6209259033203125,
+      "step": 740
+    },
+    {
+      "epoch": 0.3925673907354096,
+      "grad_norm": 63.695440464895746,
+      "learning_rate": 3.8049800956079545e-07,
+      "logits/chosen": -1.434788703918457,
+      "logits/rejected": -1.0202864408493042,
+      "logps/chosen": -409.87701416015625,
+      "logps/rejected": -506.40179443359375,
+      "loss": 0.492,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0980966091156006,
+      "rewards/margins": 0.8423215746879578,
+      "rewards/rejected": -2.940418243408203,
+      "step": 750
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 74.4694029881303,
+      "learning_rate": 3.7657936786467525e-07,
+      "logits/chosen": -1.2294615507125854,
+      "logits/rejected": -0.37707972526550293,
+      "logps/chosen": -439.51025390625,
+      "logps/rejected": -541.850830078125,
+      "loss": 0.5554,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.648507595062256,
+      "rewards/margins": 1.04387366771698,
+      "rewards/rejected": -3.6923813819885254,
+      "step": 760
+    },
+    {
+      "epoch": 0.40303585448835383,
+      "grad_norm": 71.73793938729527,
+      "learning_rate": 3.7261844968793226e-07,
+      "logits/chosen": -1.8120813369750977,
+      "logits/rejected": -1.2725474834442139,
+      "logps/chosen": -390.5841979980469,
+      "logps/rejected": -455.8675231933594,
+      "loss": 0.5758,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8539800643920898,
+      "rewards/margins": 0.7287551164627075,
+      "rewards/rejected": -2.582735300064087,
+      "step": 770
+    },
+    {
+      "epoch": 0.408270086364826,
+      "grad_norm": 39.07736477725613,
+      "learning_rate": 3.6861657794506187e-07,
+      "logits/chosen": -1.9753271341323853,
+      "logits/rejected": -1.181422233581543,
+      "logps/chosen": -334.88177490234375,
+      "logps/rejected": -404.42864990234375,
+      "loss": 0.5208,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.412209153175354,
+      "rewards/margins": 0.8599991798400879,
+      "rewards/rejected": -2.2722084522247314,
+      "step": 780
+    },
+    {
+      "epoch": 0.4135043182412981,
+      "grad_norm": 43.71312206302924,
+      "learning_rate": 3.6457508922871777e-07,
+      "logits/chosen": -1.889682412147522,
+      "logits/rejected": -1.6067005395889282,
+      "logps/chosen": -302.65997314453125,
+      "logps/rejected": -356.685302734375,
+      "loss": 0.5254,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.3029696941375732,
+      "rewards/margins": 0.5454732775688171,
+      "rewards/rejected": -1.8484432697296143,
+      "step": 790
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 55.95387505858846,
+      "learning_rate": 3.6049533336330084e-07,
+      "logits/chosen": -1.7215135097503662,
+      "logits/rejected": -1.557579755783081,
+      "logps/chosen": -316.13299560546875,
+      "logps/rejected": -418.8580017089844,
+      "loss": 0.574,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4686353206634521,
+      "rewards/margins": 0.7653969526290894,
+      "rewards/rejected": -2.234032154083252,
+      "step": 800
+    },
+    {
+      "epoch": 0.4239727819942423,
+      "grad_norm": 55.856275598760135,
+      "learning_rate": 3.56378672954129e-07,
+      "logits/chosen": -1.6764100790023804,
+      "logits/rejected": -1.513831377029419,
+      "logps/chosen": -329.6163635253906,
+      "logps/rejected": -439.32666015625,
+      "loss": 0.5538,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5339328050613403,
+      "rewards/margins": 0.7833306789398193,
+      "rewards/rejected": -2.317263126373291,
+      "step": 810
+    },
+    {
+      "epoch": 0.42920701387071447,
+      "grad_norm": 30.497969544668866,
+      "learning_rate": 3.5222648293233803e-07,
+      "logits/chosen": -1.4442280530929565,
+      "logits/rejected": -1.609626054763794,
+      "logps/chosen": -373.2001037597656,
+      "logps/rejected": -476.081787109375,
+      "loss": 0.5508,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8375556468963623,
+      "rewards/margins": 0.7855282425880432,
+      "rewards/rejected": -2.6230838298797607,
+      "step": 820
+    },
+    {
+      "epoch": 0.4344412457471866,
+      "grad_norm": 61.68802213789782,
+      "learning_rate": 3.480401500956657e-07,
+      "logits/chosen": -1.523938536643982,
+      "logits/rejected": -1.2494620084762573,
+      "logps/chosen": -351.20379638671875,
+      "logps/rejected": -404.7106018066406,
+      "loss": 0.566,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7532094717025757,
+      "rewards/margins": 0.5370227098464966,
+      "rewards/rejected": -2.2902321815490723,
+      "step": 830
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 60.810924331570874,
+      "learning_rate": 3.438210726452724e-07,
+      "logits/chosen": -1.4685875177383423,
+      "logits/rejected": -0.2091219425201416,
+      "logps/chosen": -392.4021911621094,
+      "logps/rejected": -455.44482421875,
+      "loss": 0.5227,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.634267807006836,
+      "rewards/margins": 0.8823213577270508,
+      "rewards/rejected": -2.5165891647338867,
+      "step": 840
+    },
+    {
+      "epoch": 0.44490970950013087,
+      "grad_norm": 96.20779655004723,
+      "learning_rate": 3.395706597187538e-07,
+      "logits/chosen": -1.657881498336792,
+      "logits/rejected": -1.0053375959396362,
+      "logps/chosen": -368.91693115234375,
+      "logps/rejected": -448.57928466796875,
+      "loss": 0.5498,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.6729650497436523,
+      "rewards/margins": 0.8312269449234009,
+      "rewards/rejected": -2.5041918754577637,
+      "step": 850
+    },
+    {
+      "epoch": 0.45014394137660296,
+      "grad_norm": 42.3904090971244,
+      "learning_rate": 3.3529033091949986e-07,
+      "logits/chosen": -1.2889825105667114,
+      "logits/rejected": -1.143614411354065,
+      "logps/chosen": -324.1905212402344,
+      "logps/rejected": -454.4544372558594,
+      "loss": 0.5259,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6198065280914307,
+      "rewards/margins": 0.8226202130317688,
+      "rewards/rejected": -2.4424266815185547,
+      "step": 860
+    },
+    {
+      "epoch": 0.4553781732530751,
+      "grad_norm": 58.49911093913357,
+      "learning_rate": 3.309815158425591e-07,
+      "logits/chosen": -1.6366761922836304,
+      "logits/rejected": -1.365851640701294,
+      "logps/chosen": -323.0370178222656,
+      "logps/rejected": -400.08746337890625,
+      "loss": 0.5314,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.317899465560913,
+      "rewards/margins": 0.7919186353683472,
+      "rewards/rejected": -2.1098179817199707,
+      "step": 870
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 51.25947355514214,
+      "learning_rate": 3.2664565359716536e-07,
+      "logits/chosen": -1.6139739751815796,
+      "logits/rejected": -1.3585468530654907,
+      "logps/chosen": -331.5315856933594,
+      "logps/rejected": -412.1536560058594,
+      "loss": 0.5143,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.580944299697876,
+      "rewards/margins": 0.688396155834198,
+      "rewards/rejected": -2.269340753555298,
+      "step": 880
+    },
+    {
+      "epoch": 0.46584663700601936,
+      "grad_norm": 46.47446781450997,
+      "learning_rate": 3.222841923260869e-07,
+      "logits/chosen": -1.56208074092865,
+      "logits/rejected": -1.448430061340332,
+      "logps/chosen": -320.86212158203125,
+      "logps/rejected": -417.16864013671875,
+      "loss": 0.5148,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.4867175817489624,
+      "rewards/margins": 0.8568013310432434,
+      "rewards/rejected": -2.3435187339782715,
+      "step": 890
+    },
+    {
+      "epoch": 0.4710808688824915,
+      "grad_norm": 48.96651105399355,
+      "learning_rate": 3.1789858872195887e-07,
+      "logits/chosen": -1.9190874099731445,
+      "logits/rejected": -1.5447766780853271,
+      "logps/chosen": -371.22479248046875,
+      "logps/rejected": -460.0589904785156,
+      "loss": 0.5126,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.576744794845581,
+      "rewards/margins": 0.9420223236083984,
+      "rewards/rejected": -2.5187671184539795,
+      "step": 900
+    },
+    {
+      "epoch": 0.4763151007589636,
+      "grad_norm": 52.14197578469426,
+      "learning_rate": 3.1349030754075937e-07,
+      "logits/chosen": -1.5406408309936523,
+      "logits/rejected": -1.2619167566299438,
+      "logps/chosen": -352.7300109863281,
+      "logps/rejected": -432.47027587890625,
+      "loss": 0.5743,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5983822345733643,
+      "rewards/margins": 0.7195128202438354,
+      "rewards/rejected": -2.3178951740264893,
+      "step": 910
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 40.54722236321293,
+      "learning_rate": 3.090608211125931e-07,
+      "logits/chosen": -1.6173185110092163,
+      "logits/rejected": -1.3097602128982544,
+      "logps/chosen": -330.4248046875,
+      "logps/rejected": -428.1268005371094,
+      "loss": 0.5167,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.4730908870697021,
+      "rewards/margins": 0.8812786340713501,
+      "rewards/rejected": -2.354369640350342,
+      "step": 920
+    },
+    {
+      "epoch": 0.48678356451190785,
+      "grad_norm": 72.50233844212927,
+      "learning_rate": 3.0461160884994487e-07,
+      "logits/chosen": -1.5237468481063843,
+      "logits/rejected": -1.1552776098251343,
+      "logps/chosen": -362.2909240722656,
+      "logps/rejected": -435.55718994140625,
+      "loss": 0.5268,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6922056674957275,
+      "rewards/margins": 0.7138403654098511,
+      "rewards/rejected": -2.4060463905334473,
+      "step": 930
+    },
+    {
+      "epoch": 0.49201779638838,
+      "grad_norm": 43.01313334263546,
+      "learning_rate": 3.001441567535681e-07,
+      "logits/chosen": -1.4215238094329834,
+      "logits/rejected": -0.5556894540786743,
+      "logps/chosen": -325.53814697265625,
+      "logps/rejected": -438.48602294921875,
+      "loss": 0.4948,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.556456208229065,
+      "rewards/margins": 1.1343437433242798,
+      "rewards/rejected": -2.6907999515533447,
+      "step": 940
+    },
+    {
+      "epoch": 0.49725202826485215,
+      "grad_norm": 53.39321127260436,
+      "learning_rate": 2.956599569161724e-07,
+      "logits/chosen": -1.5301493406295776,
+      "logits/rejected": -0.6811811327934265,
+      "logps/chosen": -359.93988037109375,
+      "logps/rejected": -444.7823181152344,
+      "loss": 0.504,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.7924388647079468,
+      "rewards/margins": 0.9484294652938843,
+      "rewards/rejected": -2.740868330001831,
+      "step": 950
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 54.86988246933394,
+      "learning_rate": 2.91160507024077e-07,
+      "logits/chosen": -1.3366544246673584,
+      "logits/rejected": -0.8686436414718628,
+      "logps/chosen": -387.9107360839844,
+      "logps/rejected": -483.66546630859375,
+      "loss": 0.5133,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0935609340667725,
+      "rewards/margins": 0.9254472851753235,
+      "rewards/rejected": -3.019008159637451,
+      "step": 960
+    },
+    {
+      "epoch": 0.5077204920177963,
+      "grad_norm": 40.90699971350834,
+      "learning_rate": 2.866473098569953e-07,
+      "logits/chosen": -1.3852179050445557,
+      "logits/rejected": -1.096529483795166,
+      "logps/chosen": -406.48736572265625,
+      "logps/rejected": -503.5655822753906,
+      "loss": 0.5081,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1852097511291504,
+      "rewards/margins": 0.9200779795646667,
+      "rewards/rejected": -3.105287551879883,
+      "step": 970
+    },
+    {
+      "epoch": 0.5129547238942685,
+      "grad_norm": 58.095451640836856,
+      "learning_rate": 2.8212187278611905e-07,
+      "logits/chosen": -1.689866065979004,
+      "logits/rejected": -1.0209802389144897,
+      "logps/chosen": -391.45513916015625,
+      "logps/rejected": -496.5826110839844,
+      "loss": 0.5035,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.8062623739242554,
+      "rewards/margins": 0.9972459077835083,
+      "rewards/rejected": -2.8035082817077637,
+      "step": 980
+    },
+    {
+      "epoch": 0.5181889557707406,
+      "grad_norm": 59.318676556528416,
+      "learning_rate": 2.775857072706684e-07,
+      "logits/chosen": -1.3584972620010376,
+      "logits/rejected": -1.3198730945587158,
+      "logps/chosen": -378.9326171875,
+      "logps/rejected": -501.0203552246094,
+      "loss": 0.5252,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.007432460784912,
+      "rewards/margins": 0.9733474850654602,
+      "rewards/rejected": -2.9807801246643066,
+      "step": 990
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 71.87530393074267,
+      "learning_rate": 2.7304032835307667e-07,
+      "logits/chosen": -1.4289054870605469,
+      "logits/rejected": -1.1081018447875977,
+      "logps/chosen": -383.5538024902344,
+      "logps/rejected": -441.691650390625,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.0912938117980957,
+      "rewards/margins": 0.5263177156448364,
+      "rewards/rejected": -2.6176114082336426,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "eval_logits/chosen": -0.6161556839942932,
+      "eval_logits/rejected": -0.05726008489727974,
+      "eval_logps/chosen": -376.86456298828125,
+      "eval_logps/rejected": -464.8497009277344,
+      "eval_loss": 0.5098932385444641,
+      "eval_rewards/accuracies": 0.7160000205039978,
+      "eval_rewards/chosen": -1.9184350967407227,
+      "eval_rewards/margins": 0.838153064250946,
+      "eval_rewards/rejected": -2.7565884590148926,
+      "eval_runtime": 316.7389,
+      "eval_samples_per_second": 6.314,
+      "eval_steps_per_second": 0.789,
+      "step": 1000
+    },
+    {
+      "epoch": 0.528657419523685,
+      "grad_norm": 34.830235400374406,
+      "learning_rate": 2.6848725415297884e-07,
+      "logits/chosen": -1.6549265384674072,
+      "logits/rejected": -1.5296745300292969,
+      "logps/chosen": -367.93243408203125,
+      "logps/rejected": -463.02581787109375,
+      "loss": 0.5217,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8079150915145874,
+      "rewards/margins": 0.8082486391067505,
+      "rewards/rejected": -2.616163730621338,
+      "step": 1010
+    },
+    {
+      "epoch": 0.533891651400157,
+      "grad_norm": 46.15714344837178,
+      "learning_rate": 2.6392800536017183e-07,
+      "logits/chosen": -1.3246982097625732,
+      "logits/rejected": -1.457916498184204,
+      "logps/chosen": -347.6453552246094,
+      "logps/rejected": -456.356689453125,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6062390804290771,
+      "rewards/margins": 0.8783868551254272,
+      "rewards/rejected": -2.484626054763794,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5391258832766291,
+      "grad_norm": 40.29007295543721,
+      "learning_rate": 2.59364104726716e-07,
+      "logits/chosen": -1.495118498802185,
+      "logits/rejected": -1.118583083152771,
+      "logps/chosen": -361.8853454589844,
+      "logps/rejected": -440.2359924316406,
+      "loss": 0.4939,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8378098011016846,
+      "rewards/margins": 0.7821410894393921,
+      "rewards/rejected": -2.619950771331787,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 41.4595455203303,
+      "learning_rate": 2.547970765583491e-07,
+      "logits/chosen": -1.4750334024429321,
+      "logits/rejected": -1.2206333875656128,
+      "logps/chosen": -345.68243408203125,
+      "logps/rejected": -457.96417236328125,
+      "loss": 0.5369,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.8482234477996826,
+      "rewards/margins": 0.8760908246040344,
+      "rewards/rejected": -2.7243142127990723,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5495943470295734,
+      "grad_norm": 69.76278032549452,
+      "learning_rate": 2.502284462053799e-07,
+      "logits/chosen": -1.6379356384277344,
+      "logits/rejected": -1.1788249015808105,
+      "logps/chosen": -364.0078125,
+      "logps/rejected": -483.25750732421875,
+      "loss": 0.5505,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.7867047786712646,
+      "rewards/margins": 0.9141599535942078,
+      "rewards/rejected": -2.700864553451538,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5548285789060455,
+      "grad_norm": 43.473418630985485,
+      "learning_rate": 2.4565973955323374e-07,
+      "logits/chosen": -1.9050061702728271,
+      "logits/rejected": -0.7371460795402527,
+      "logps/chosen": -347.5364990234375,
+      "logps/rejected": -419.16436767578125,
+      "loss": 0.5246,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.645029067993164,
+      "rewards/margins": 0.8056646585464478,
+      "rewards/rejected": -2.4506936073303223,
+      "step": 1060
+    },
+    {
+      "epoch": 0.5600628107825176,
+      "grad_norm": 44.18467471903625,
+      "learning_rate": 2.410924825128195e-07,
+      "logits/chosen": -1.677795648574829,
+      "logits/rejected": -1.0934385061264038,
+      "logps/chosen": -336.7928161621094,
+      "logps/rejected": -414.9443359375,
+      "loss": 0.5456,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.7231500148773193,
+      "rewards/margins": 0.6935104727745056,
+      "rewards/rejected": -2.416660785675049,
+      "step": 1070
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 43.21905358731484,
+      "learning_rate": 2.365282005108875e-07,
+      "logits/chosen": -1.9169113636016846,
+      "logits/rejected": -1.1039310693740845,
+      "logps/chosen": -326.3893127441406,
+      "logps/rejected": -395.2596435546875,
+      "loss": 0.5318,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3917843103408813,
+      "rewards/margins": 0.8932709693908691,
+      "rewards/rejected": -2.28505539894104,
+      "step": 1080
+    },
+    {
+      "epoch": 0.5705312745354619,
+      "grad_norm": 57.74038702140702,
+      "learning_rate": 2.319684179805491e-07,
+      "logits/chosen": -1.7448060512542725,
+      "logits/rejected": -0.4081326425075531,
+      "logps/chosen": -359.01214599609375,
+      "logps/rejected": -431.38531494140625,
+      "loss": 0.5478,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.73810613155365,
+      "rewards/margins": 0.8466591835021973,
+      "rewards/rejected": -2.5847649574279785,
+      "step": 1090
+    },
+    {
+      "epoch": 0.575765506411934,
+      "grad_norm": 47.91534163404264,
+      "learning_rate": 2.2741465785212902e-07,
+      "logits/chosen": -1.360697627067566,
+      "logits/rejected": -1.1678688526153564,
+      "logps/chosen": -341.6931457519531,
+      "logps/rejected": -434.61749267578125,
+      "loss": 0.5126,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7018409967422485,
+      "rewards/margins": 0.6925634741783142,
+      "rewards/rejected": -2.394404649734497,
+      "step": 1100
+    },
+    {
+      "epoch": 0.5809997382884062,
+      "grad_norm": 45.37428786887181,
+      "learning_rate": 2.2286844104451843e-07,
+      "logits/chosen": -1.6492805480957031,
+      "logits/rejected": -1.0351827144622803,
+      "logps/chosen": -381.6548156738281,
+      "logps/rejected": -453.56561279296875,
+      "loss": 0.5463,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7667547464370728,
+      "rewards/margins": 0.7139405012130737,
+      "rewards/rejected": -2.4806952476501465,
+      "step": 1110
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 55.038838111114046,
+      "learning_rate": 2.183312859572008e-07,
+      "logits/chosen": -1.5666944980621338,
+      "logits/rejected": -1.518080711364746,
+      "logps/chosen": -349.3604431152344,
+      "logps/rejected": -461.66033935546875,
+      "loss": 0.5142,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.716058373451233,
+      "rewards/margins": 0.9028706550598145,
+      "rewards/rejected": -2.618929147720337,
+      "step": 1120
+    },
+    {
+      "epoch": 0.5914682020413504,
+      "grad_norm": 51.86256999226559,
+      "learning_rate": 2.138047079631184e-07,
+      "logits/chosen": -1.71830153465271,
+      "logits/rejected": -1.3731257915496826,
+      "logps/chosen": -348.7271423339844,
+      "logps/rejected": -429.04730224609375,
+      "loss": 0.5304,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.4229063987731934,
+      "rewards/margins": 0.8486738204956055,
+      "rewards/rejected": -2.271580219268799,
+      "step": 1130
+    },
+    {
+      "epoch": 0.5967024339178225,
+      "grad_norm": 61.89587232832013,
+      "learning_rate": 2.0929021890255068e-07,
+      "logits/chosen": -1.7627557516098022,
+      "logits/rejected": -0.9542855024337769,
+      "logps/chosen": -336.31402587890625,
+      "logps/rejected": -434.7019958496094,
+      "loss": 0.4759,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.5431474447250366,
+      "rewards/margins": 1.0138742923736572,
+      "rewards/rejected": -2.5570216178894043,
+      "step": 1140
+    },
+    {
+      "epoch": 0.6019366657942947,
+      "grad_norm": 40.16605369465375,
+      "learning_rate": 2.0478932657817102e-07,
+      "logits/chosen": -1.7182365655899048,
+      "logits/rejected": -1.2019648551940918,
+      "logps/chosen": -383.36444091796875,
+      "logps/rejected": -474.964599609375,
+      "loss": 0.5167,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7786900997161865,
+      "rewards/margins": 0.8999784588813782,
+      "rewards/rejected": -2.67866849899292,
+      "step": 1150
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 65.12157906582378,
+      "learning_rate": 2.0030353425145374e-07,
+      "logits/chosen": -1.5357849597930908,
+      "logits/rejected": -0.887830913066864,
+      "logps/chosen": -351.23516845703125,
+      "logps/rejected": -427.96795654296875,
+      "loss": 0.5519,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.722428560256958,
+      "rewards/margins": 0.9013321995735168,
+      "rewards/rejected": -2.62376070022583,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6124051295472389,
+      "grad_norm": 56.641784845018215,
+      "learning_rate": 1.9583434014059635e-07,
+      "logits/chosen": -1.632693886756897,
+      "logits/rejected": -1.4601401090621948,
+      "logps/chosen": -347.542236328125,
+      "logps/rejected": -451.15875244140625,
+      "loss": 0.5421,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6873490810394287,
+      "rewards/margins": 0.8980420827865601,
+      "rewards/rejected": -2.5853912830352783,
+      "step": 1170
+    },
+    {
+      "epoch": 0.6176393614237111,
+      "grad_norm": 45.373706821931066,
+      "learning_rate": 1.9138323692012733e-07,
+      "logits/chosen": -1.8676795959472656,
+      "logits/rejected": -1.3623721599578857,
+      "logps/chosen": -318.84417724609375,
+      "logps/rejected": -431.2315979003906,
+      "loss": 0.4796,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.421907901763916,
+      "rewards/margins": 0.911504864692688,
+      "rewards/rejected": -2.3334126472473145,
+      "step": 1180
+    },
+    {
+      "epoch": 0.6228735933001832,
+      "grad_norm": 33.99709051781791,
+      "learning_rate": 1.8695171122236442e-07,
+      "logits/chosen": -1.6899328231811523,
+      "logits/rejected": -1.231277585029602,
+      "logps/chosen": -341.48797607421875,
+      "logps/rejected": -474.08343505859375,
+      "loss": 0.4651,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.5275802612304688,
+      "rewards/margins": 1.2658531665802002,
+      "rewards/rejected": -2.793433427810669,
+      "step": 1190
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 59.87117531330853,
+      "learning_rate": 1.8254124314089223e-07,
+      "logits/chosen": -1.715743064880371,
+      "logits/rejected": -1.2481369972229004,
+      "logps/chosen": -357.3684997558594,
+      "logps/rejected": -465.179443359375,
+      "loss": 0.4937,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.7480823993682861,
+      "rewards/margins": 0.9352186322212219,
+      "rewards/rejected": -2.6833012104034424,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6333420570531274,
+      "grad_norm": 47.15786058024146,
+      "learning_rate": 1.7815330573622205e-07,
+      "logits/chosen": -1.7806997299194336,
+      "logits/rejected": -1.0503987073898315,
+      "logps/chosen": -366.83319091796875,
+      "logps/rejected": -447.2417907714844,
+      "loss": 0.5129,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8660345077514648,
+      "rewards/margins": 0.938534140586853,
+      "rewards/rejected": -2.8045687675476074,
+      "step": 1210
+    },
+    {
+      "epoch": 0.6385762889295996,
+      "grad_norm": 59.66066585989714,
+      "learning_rate": 1.7378936454380274e-07,
+      "logits/chosen": -1.665154218673706,
+      "logits/rejected": -1.2627506256103516,
+      "logps/chosen": -375.3360290527344,
+      "logps/rejected": -451.8492736816406,
+      "loss": 0.5141,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.9925143718719482,
+      "rewards/margins": 0.8351390957832336,
+      "rewards/rejected": -2.827653408050537,
+      "step": 1220
+    },
+    {
+      "epoch": 0.6438105208060717,
+      "grad_norm": 41.626090508354544,
+      "learning_rate": 1.694508770845427e-07,
+      "logits/chosen": -1.8555021286010742,
+      "logits/rejected": -1.1900012493133545,
+      "logps/chosen": -381.927490234375,
+      "logps/rejected": -469.22900390625,
+      "loss": 0.481,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.8637516498565674,
+      "rewards/margins": 0.8393726348876953,
+      "rewards/rejected": -2.7031240463256836,
+      "step": 1230
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 46.575209941519326,
+      "learning_rate": 1.651392923780105e-07,
+      "logits/chosen": -1.7130018472671509,
+      "logits/rejected": -1.2077395915985107,
+      "logps/chosen": -395.5782775878906,
+      "logps/rejected": -510.3377380371094,
+      "loss": 0.5015,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0861849784851074,
+      "rewards/margins": 1.0165139436721802,
+      "rewards/rejected": -3.102698802947998,
+      "step": 1240
+    },
+    {
+      "epoch": 0.654278984559016,
+      "grad_norm": 37.25325816311792,
+      "learning_rate": 1.6085605045847367e-07,
+      "logits/chosen": -1.3418257236480713,
+      "logits/rejected": -0.8439585566520691,
+      "logps/chosen": -367.1421813964844,
+      "logps/rejected": -509.10919189453125,
+      "loss": 0.4888,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.0315098762512207,
+      "rewards/margins": 1.2717547416687012,
+      "rewards/rejected": -3.303264617919922,
+      "step": 1250
+    },
+    {
+      "epoch": 0.6595132164354881,
+      "grad_norm": 56.54421548734596,
+      "learning_rate": 1.5660258189393944e-07,
+      "logits/chosen": -1.6082178354263306,
+      "logits/rejected": -0.8709881901741028,
+      "logps/chosen": -418.84100341796875,
+      "logps/rejected": -491.2918395996094,
+      "loss": 0.4837,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.1445069313049316,
+      "rewards/margins": 0.9897298812866211,
+      "rewards/rejected": -3.1342368125915527,
+      "step": 1260
+    },
+    {
+      "epoch": 0.6647474483119602,
+      "grad_norm": 58.872726779228444,
+      "learning_rate": 1.5238030730835577e-07,
+      "logits/chosen": -1.4633188247680664,
+      "logits/rejected": -1.0070345401763916,
+      "logps/chosen": -417.1031188964844,
+      "logps/rejected": -512.1041870117188,
+      "loss": 0.4794,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.1558361053466797,
+      "rewards/margins": 1.0906795263290405,
+      "rewards/rejected": -3.246515989303589,
+      "step": 1270
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 81.68062237565607,
+      "learning_rate": 1.4819063690713564e-07,
+      "logits/chosen": -1.230272650718689,
+      "logits/rejected": -1.1420851945877075,
+      "logps/chosen": -400.37384033203125,
+      "logps/rejected": -501.90203857421875,
+      "loss": 0.4836,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.241870403289795,
+      "rewards/margins": 0.9584578275680542,
+      "rewards/rejected": -3.2003283500671387,
+      "step": 1280
+    },
+    {
+      "epoch": 0.6752159120649045,
+      "grad_norm": 43.4045980317099,
+      "learning_rate": 1.4403497000615883e-07,
+      "logits/chosen": -1.5301976203918457,
+      "logits/rejected": -1.0589085817337036,
+      "logps/chosen": -406.1276550292969,
+      "logps/rejected": -488.4261779785156,
+      "loss": 0.5306,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.061319589614868,
+      "rewards/margins": 0.8899547457695007,
+      "rewards/rejected": -2.9512743949890137,
+      "step": 1290
+    },
+    {
+      "epoch": 0.6804501439413766,
+      "grad_norm": 89.57498346480389,
+      "learning_rate": 1.3991469456441272e-07,
+      "logits/chosen": -1.6822435855865479,
+      "logits/rejected": -1.314344882965088,
+      "logps/chosen": -366.78118896484375,
+      "logps/rejected": -476.983642578125,
+      "loss": 0.4653,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9514825344085693,
+      "rewards/margins": 0.9372948408126831,
+      "rewards/rejected": -2.888777256011963,
+      "step": 1300
+    },
+    {
+      "epoch": 0.6856843758178487,
+      "grad_norm": 79.22800819366738,
+      "learning_rate": 1.358311867204244e-07,
+      "logits/chosen": -1.3850901126861572,
+      "logits/rejected": -1.0397329330444336,
+      "logps/chosen": -410.082763671875,
+      "logps/rejected": -514.4329833984375,
+      "loss": 0.5204,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.1403348445892334,
+      "rewards/margins": 0.8943174481391907,
+      "rewards/rejected": -3.0346522331237793,
+      "step": 1310
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 38.40836601776419,
+      "learning_rate": 1.3178581033264216e-07,
+      "logits/chosen": -1.7234236001968384,
+      "logits/rejected": -1.0658862590789795,
+      "logps/chosen": -384.4350280761719,
+      "logps/rejected": -510.3180236816406,
+      "loss": 0.4734,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.981745719909668,
+      "rewards/margins": 1.077263355255127,
+      "rewards/rejected": -3.059009075164795,
+      "step": 1320
+    },
+    {
+      "epoch": 0.696152839570793,
+      "grad_norm": 64.64336518776386,
+      "learning_rate": 1.2777991652391757e-07,
+      "logits/chosen": -1.686440110206604,
+      "logits/rejected": -1.139702558517456,
+      "logps/chosen": -364.24102783203125,
+      "logps/rejected": -508.68310546875,
+      "loss": 0.4863,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.8615505695343018,
+      "rewards/margins": 1.3014369010925293,
+      "rewards/rejected": -3.162987232208252,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7013870714472651,
+      "grad_norm": 54.064436812936954,
+      "learning_rate": 1.2381484323024178e-07,
+      "logits/chosen": -1.5776959657669067,
+      "logits/rejected": -1.3716309070587158,
+      "logps/chosen": -381.18890380859375,
+      "logps/rejected": -485.0956115722656,
+      "loss": 0.4748,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.0563852787017822,
+      "rewards/margins": 0.8801882863044739,
+      "rewards/rejected": -2.9365735054016113,
+      "step": 1340
+    },
+    {
+      "epoch": 0.7066213033237373,
+      "grad_norm": 71.26268968336478,
+      "learning_rate": 1.1989191475388516e-07,
+      "logits/chosen": -1.5418208837509155,
+      "logits/rejected": -1.3163645267486572,
+      "logps/chosen": -389.85479736328125,
+      "logps/rejected": -469.4613342285156,
+      "loss": 0.5114,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9432039260864258,
+      "rewards/margins": 0.8496206402778625,
+      "rewards/rejected": -2.7928245067596436,
+      "step": 1350
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 47.16713356681667,
+      "learning_rate": 1.1601244132109179e-07,
+      "logits/chosen": -1.547049641609192,
+      "logits/rejected": -1.1506636142730713,
+      "logps/chosen": -378.78546142578125,
+      "logps/rejected": -475.17340087890625,
+      "loss": 0.5175,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9611685276031494,
+      "rewards/margins": 0.8856902122497559,
+      "rewards/rejected": -2.8468587398529053,
+      "step": 1360
+    },
+    {
+      "epoch": 0.7170897670766815,
+      "grad_norm": 57.49724175574713,
+      "learning_rate": 1.1217771864447395e-07,
+      "logits/chosen": -1.5181536674499512,
+      "logits/rejected": -1.0610802173614502,
+      "logps/chosen": -382.1415710449219,
+      "logps/rejected": -458.43310546875,
+      "loss": 0.446,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9159305095672607,
+      "rewards/margins": 0.8874963521957397,
+      "rewards/rejected": -2.803427219390869,
+      "step": 1370
+    },
+    {
+      "epoch": 0.7223239989531536,
+      "grad_norm": 48.43604137330991,
+      "learning_rate": 1.0838902749025499e-07,
+      "logits/chosen": -1.5803115367889404,
+      "logits/rejected": -0.9147381782531738,
+      "logps/chosen": -364.0269470214844,
+      "logps/rejected": -485.8170471191406,
+      "loss": 0.4886,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.7391821146011353,
+      "rewards/margins": 1.2611137628555298,
+      "rewards/rejected": -3.000295639038086,
+      "step": 1380
+    },
+    {
+      "epoch": 0.7275582308296258,
+      "grad_norm": 46.02896607743019,
+      "learning_rate": 1.0464763325050358e-07,
+      "logits/chosen": -1.389105200767517,
+      "logits/rejected": -1.2149088382720947,
+      "logps/chosen": -363.1921081542969,
+      "logps/rejected": -481.0618591308594,
+      "loss": 0.4901,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.8936526775360107,
+      "rewards/margins": 1.019033670425415,
+      "rewards/rejected": -2.912686586380005,
+      "step": 1390
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 47.49065837586764,
+      "learning_rate": 1.0095478552050346e-07,
+      "logits/chosen": -1.6921743154525757,
+      "logits/rejected": -1.1181484460830688,
+      "logps/chosen": -369.4226989746094,
+      "logps/rejected": -469.757568359375,
+      "loss": 0.5117,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.803931474685669,
+      "rewards/margins": 0.8779823184013367,
+      "rewards/rejected": -2.6819140911102295,
+      "step": 1400
+    },
+    {
+      "epoch": 0.73802669458257,
+      "grad_norm": 51.70967904748882,
+      "learning_rate": 9.731171768139806e-08,
+      "logits/chosen": -1.5017741918563843,
+      "logits/rejected": -0.8476959466934204,
+      "logps/chosen": -366.8438415527344,
+      "logps/rejected": -477.4512634277344,
+      "loss": 0.5513,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8564783334732056,
+      "rewards/margins": 1.0491094589233398,
+      "rewards/rejected": -2.905587673187256,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7432609264590422,
+      "grad_norm": 38.12827677711878,
+      "learning_rate": 9.37196464882522e-08,
+      "logits/chosen": -1.7628695964813232,
+      "logits/rejected": -0.9748938679695129,
+      "logps/chosen": -358.43377685546875,
+      "logps/rejected": -426.41839599609375,
+      "loss": 0.5369,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.729318380355835,
+      "rewards/margins": 0.7932037711143494,
+      "rewards/rejected": -2.522522211074829,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7484951583355143,
+      "grad_norm": 54.47585201435841,
+      "learning_rate": 9.017977166366444e-08,
+      "logits/chosen": -1.7095372676849365,
+      "logits/rejected": -1.3382246494293213,
+      "logps/chosen": -386.80419921875,
+      "logps/rejected": -485.66217041015625,
+      "loss": 0.4759,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.8498786687850952,
+      "rewards/margins": 0.956847071647644,
+      "rewards/rejected": -2.8067257404327393,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 55.994908017980144,
+      "learning_rate": 8.669327549707095e-08,
+      "logits/chosen": -1.8079534769058228,
+      "logits/rejected": -1.6061460971832275,
+      "logps/chosen": -372.0968017578125,
+      "logps/rejected": -449.3272399902344,
+      "loss": 0.4883,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.8120940923690796,
+      "rewards/margins": 0.7753244042396545,
+      "rewards/rejected": -2.587418794631958,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7589636220884585,
+      "grad_norm": 44.71580454221332,
+      "learning_rate": 8.326132244986931e-08,
+      "logits/chosen": -1.501529335975647,
+      "logits/rejected": -1.167120337486267,
+      "logps/chosen": -371.7989807128906,
+      "logps/rejected": -466.7962951660156,
+      "loss": 0.4955,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9070402383804321,
+      "rewards/margins": 0.8023300170898438,
+      "rewards/rejected": -2.7093701362609863,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7641978539649307,
+      "grad_norm": 47.918962412956276,
+      "learning_rate": 7.988505876649862e-08,
+      "logits/chosen": -1.6484168767929077,
+      "logits/rejected": -1.049037218093872,
+      "logps/chosen": -387.1507263183594,
+      "logps/rejected": -471.23455810546875,
+      "loss": 0.5227,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.9400913715362549,
+      "rewards/margins": 0.9767180681228638,
+      "rewards/rejected": -2.916809320449829,
+      "step": 1460
+    },
+    {
+      "epoch": 0.7694320858414028,
+      "grad_norm": 44.681178148336286,
+      "learning_rate": 7.656561209160248e-08,
+      "logits/chosen": -1.5934867858886719,
+      "logits/rejected": -0.9533841013908386,
+      "logps/chosen": -415.989013671875,
+      "logps/rejected": -503.83038330078125,
+      "loss": 0.5124,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.1351356506347656,
+      "rewards/margins": 1.072280764579773,
+      "rewards/rejected": -3.2074170112609863,
+      "step": 1470
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 42.957667070339845,
+      "learning_rate": 7.330409109340562e-08,
+      "logits/chosen": -1.6417922973632812,
+      "logits/rejected": -0.9782828092575073,
+      "logps/chosen": -368.1243591308594,
+      "logps/rejected": -457.6202697753906,
+      "loss": 0.53,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.979144811630249,
+      "rewards/margins": 0.9068864583969116,
+      "rewards/rejected": -2.88603138923645,
+      "step": 1480
+    },
+    {
+      "epoch": 0.7799005495943471,
+      "grad_norm": 50.32703939295202,
+      "learning_rate": 7.010158509342681e-08,
+      "logits/chosen": -1.492157220840454,
+      "logits/rejected": -1.0186296701431274,
+      "logps/chosen": -405.69769287109375,
+      "logps/rejected": -512.1472778320312,
+      "loss": 0.4922,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.2098875045776367,
+      "rewards/margins": 0.9893035888671875,
+      "rewards/rejected": -3.199190855026245,
+      "step": 1490
+    },
+    {
+      "epoch": 0.7851347814708192,
+      "grad_norm": 46.95925974979752,
+      "learning_rate": 6.695916370265527e-08,
+      "logits/chosen": -1.5276159048080444,
+      "logits/rejected": -1.1085699796676636,
+      "logps/chosen": -377.8671875,
+      "logps/rejected": -545.2662353515625,
+      "loss": 0.4707,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.0651357173919678,
+      "rewards/margins": 1.2801060676574707,
+      "rewards/rejected": -3.3452422618865967,
+      "step": 1500
+    },
+    {
+      "epoch": 0.7851347814708192,
+      "eval_logits/chosen": -0.6702331304550171,
+      "eval_logits/rejected": -0.05479540303349495,
+      "eval_logps/chosen": -403.77130126953125,
+      "eval_logps/rejected": -503.5488586425781,
+      "eval_loss": 0.500029981136322,
+      "eval_rewards/accuracies": 0.7319999933242798,
+      "eval_rewards/chosen": -2.1875030994415283,
+      "eval_rewards/margins": 0.9560768604278564,
+      "eval_rewards/rejected": -3.1435797214508057,
+      "eval_runtime": 316.8429,
+      "eval_samples_per_second": 6.312,
+      "eval_steps_per_second": 0.789,
+      "step": 1500
+    },
+    {
+      "epoch": 0.7903690133472913,
+      "grad_norm": 54.6800615563418,
+      "learning_rate": 6.387787646430853e-08,
+      "logits/chosen": -1.6765468120574951,
+      "logits/rejected": -0.902447521686554,
+      "logps/chosen": -411.84197998046875,
+      "logps/rejected": -520.2639770507812,
+      "loss": 0.5386,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.209589719772339,
+      "rewards/margins": 1.2369428873062134,
+      "rewards/rejected": -3.4465324878692627,
+      "step": 1510
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 39.76345106053052,
+      "learning_rate": 6.0858752503294e-08,
+      "logits/chosen": -1.9017261266708374,
+      "logits/rejected": -1.027804970741272,
+      "logps/chosen": -375.6845397949219,
+      "logps/rejected": -452.0986328125,
+      "loss": 0.524,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9550809860229492,
+      "rewards/margins": 0.9305828213691711,
+      "rewards/rejected": -2.8856637477874756,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8008374771002356,
+      "grad_norm": 58.54818603530255,
+      "learning_rate": 5.7902800182489385e-08,
+      "logits/chosen": -1.340591311454773,
+      "logits/rejected": -0.8900511860847473,
+      "logps/chosen": -382.7643737792969,
+      "logps/rejected": -484.4600524902344,
+      "loss": 0.4566,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.102140188217163,
+      "rewards/margins": 0.9162777066230774,
+      "rewards/rejected": -3.018418073654175,
+      "step": 1530
+    },
+    {
+      "epoch": 0.8060717089767077,
+      "grad_norm": 31.245998478258013,
+      "learning_rate": 5.5011006765957604e-08,
+      "logits/chosen": -1.618656873703003,
+      "logits/rejected": -1.1801456212997437,
+      "logps/chosen": -419.6280822753906,
+      "logps/rejected": -499.01751708984375,
+      "loss": 0.4916,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.230376720428467,
+      "rewards/margins": 0.7931150197982788,
+      "rewards/rejected": -3.023491621017456,
+      "step": 1540
+    },
+    {
+      "epoch": 0.8113059408531798,
+      "grad_norm": 51.33396122501131,
+      "learning_rate": 5.218433808920883e-08,
+      "logits/chosen": -1.60371994972229,
+      "logits/rejected": -1.1156429052352905,
+      "logps/chosen": -398.92974853515625,
+      "logps/rejected": -472.16973876953125,
+      "loss": 0.5013,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.029679775238037,
+      "rewards/margins": 0.9714105725288391,
+      "rewards/rejected": -3.0010905265808105,
+      "step": 1550
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 39.552994793370026,
+      "learning_rate": 4.942373823661927e-08,
+      "logits/chosen": -1.6282466650009155,
+      "logits/rejected": -0.91362464427948,
+      "logps/chosen": -435.72216796875,
+      "logps/rejected": -547.359619140625,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.306370496749878,
+      "rewards/margins": 0.9918804168701172,
+      "rewards/rejected": -3.298250913619995,
+      "step": 1560
+    },
+    {
+      "epoch": 0.821774404606124,
+      "grad_norm": 60.08870167062165,
+      "learning_rate": 4.6730129226114354e-08,
+      "logits/chosen": -1.6371328830718994,
+      "logits/rejected": -0.6333679556846619,
+      "logps/chosen": -426.95703125,
+      "logps/rejected": -538.3595581054688,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.2964425086975098,
+      "rewards/margins": 1.1182937622070312,
+      "rewards/rejected": -3.414735794067383,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8270086364825961,
+      "grad_norm": 46.54218001272997,
+      "learning_rate": 4.41044107012227e-08,
+      "logits/chosen": -1.46670401096344,
+      "logits/rejected": -1.1195374727249146,
+      "logps/chosen": -394.80078125,
+      "logps/rejected": -524.2930297851562,
+      "loss": 0.5174,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.0618937015533447,
+      "rewards/margins": 1.137542486190796,
+      "rewards/rejected": -3.1994361877441406,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8322428683590684,
+      "grad_norm": 61.54813763490487,
+      "learning_rate": 4.1547459630601966e-08,
+      "logits/chosen": -1.5916258096694946,
+      "logits/rejected": -0.7485260963439941,
+      "logps/chosen": -393.9290771484375,
+      "logps/rejected": -469.08026123046875,
+      "loss": 0.488,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.228323459625244,
+      "rewards/margins": 0.8031214475631714,
+      "rewards/rejected": -3.031445026397705,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 47.454976444510464,
+      "learning_rate": 3.9060130015138857e-08,
+      "logits/chosen": -1.6673399209976196,
+      "logits/rejected": -0.9725383520126343,
+      "logps/chosen": -391.20428466796875,
+      "logps/rejected": -521.3951416015625,
+      "loss": 0.5182,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.1082262992858887,
+      "rewards/margins": 1.1958465576171875,
+      "rewards/rejected": -3.304072856903076,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8427113321120125,
+      "grad_norm": 49.49623357781679,
+      "learning_rate": 3.664325260271953e-08,
+      "logits/chosen": -1.785679578781128,
+      "logits/rejected": -1.3486297130584717,
+      "logps/chosen": -425.0037536621094,
+      "logps/rejected": -499.3848571777344,
+      "loss": 0.5221,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9705803394317627,
+      "rewards/margins": 1.0100176334381104,
+      "rewards/rejected": -2.980597972869873,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8479455639884846,
+      "grad_norm": 46.41346156870081,
+      "learning_rate": 3.429763461076676e-08,
+      "logits/chosen": -1.5286333560943604,
+      "logits/rejected": -1.219725251197815,
+      "logps/chosen": -410.5511169433594,
+      "logps/rejected": -490.86669921875,
+      "loss": 0.499,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.219921112060547,
+      "rewards/margins": 0.7538778185844421,
+      "rewards/rejected": -2.973799228668213,
+      "step": 1620
+    },
+    {
+      "epoch": 0.8531797958649568,
+      "grad_norm": 48.04705054674233,
+      "learning_rate": 3.202405945663555e-08,
+      "logits/chosen": -1.2301688194274902,
+      "logits/rejected": -1.1187329292297363,
+      "logps/chosen": -398.3143615722656,
+      "logps/rejected": -471.55010986328125,
+      "loss": 0.5216,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.1741843223571777,
+      "rewards/margins": 0.7370268702507019,
+      "rewards/rejected": -2.9112114906311035,
+      "step": 1630
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 55.340731067768736,
+      "learning_rate": 2.9823286495958556e-08,
+      "logits/chosen": -1.6326100826263428,
+      "logits/rejected": -1.1603213548660278,
+      "logps/chosen": -397.24005126953125,
+      "logps/rejected": -491.3506774902344,
+      "loss": 0.5211,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9936778545379639,
+      "rewards/margins": 1.0326869487762451,
+      "rewards/rejected": -3.026364803314209,
+      "step": 1640
+    },
+    {
+      "epoch": 0.863648259617901,
+      "grad_norm": 50.97109608180561,
+      "learning_rate": 2.769605076902695e-08,
+      "logits/chosen": -1.5886093378067017,
+      "logits/rejected": -1.497037649154663,
+      "logps/chosen": -364.107666015625,
+      "logps/rejected": -491.516357421875,
+      "loss": 0.5201,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.014535903930664,
+      "rewards/margins": 0.9813510179519653,
+      "rewards/rejected": -2.9958865642547607,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8688824914943732,
+      "grad_norm": 79.38139421221125,
+      "learning_rate": 2.5643062755293403e-08,
+      "logits/chosen": -1.504066824913025,
+      "logits/rejected": -0.8931864500045776,
+      "logps/chosen": -391.62713623046875,
+      "logps/rejected": -493.92425537109375,
+      "loss": 0.5236,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.1034412384033203,
+      "rewards/margins": 0.948072075843811,
+      "rewards/rejected": -3.051513195037842,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8741167233708453,
+      "grad_norm": 51.56700786640102,
+      "learning_rate": 2.366500813607733e-08,
+      "logits/chosen": -1.5938122272491455,
+      "logits/rejected": -1.2632464170455933,
+      "logps/chosen": -385.2617492675781,
+      "logps/rejected": -489.09228515625,
+      "loss": 0.4905,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.0015676021575928,
+      "rewards/margins": 1.0208126306533813,
+      "rewards/rejected": -3.0223803520202637,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 57.01702831334661,
+      "learning_rate": 2.176254756555329e-08,
+      "logits/chosen": -1.5943069458007812,
+      "logits/rejected": -1.2856522798538208,
+      "logps/chosen": -391.60736083984375,
+      "logps/rejected": -528.451171875,
+      "loss": 0.4412,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9151815176010132,
+      "rewards/margins": 1.0721461772918701,
+      "rewards/rejected": -2.987327814102173,
+      "step": 1680
+    },
+    {
+      "epoch": 0.8845851871237895,
+      "grad_norm": 52.04207661692601,
+      "learning_rate": 1.9936316450097468e-08,
+      "logits/chosen": -1.6583569049835205,
+      "logits/rejected": -1.2720321416854858,
+      "logps/chosen": -375.07989501953125,
+      "logps/rejected": -463.97015380859375,
+      "loss": 0.48,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.0725209712982178,
+      "rewards/margins": 0.813088595867157,
+      "rewards/rejected": -2.8856096267700195,
+      "step": 1690
+    },
+    {
+      "epoch": 0.8898194190002617,
+      "grad_norm": 56.72817214823504,
+      "learning_rate": 1.8186924736067477e-08,
+      "logits/chosen": -1.6529877185821533,
+      "logits/rejected": -1.2598158121109009,
+      "logps/chosen": -395.845703125,
+      "logps/rejected": -497.45574951171875,
+      "loss": 0.4753,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.0222179889678955,
+      "rewards/margins": 1.0427659749984741,
+      "rewards/rejected": -3.06498384475708,
+      "step": 1700
+    },
+    {
+      "epoch": 0.8950536508767338,
+      "grad_norm": 60.40350237801338,
+      "learning_rate": 1.651495670608488e-08,
+      "logits/chosen": -1.6280845403671265,
+      "logits/rejected": -1.0049400329589844,
+      "logps/chosen": -408.4595642089844,
+      "logps/rejected": -528.3615112304688,
+      "loss": 0.4851,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.1130669116973877,
+      "rewards/margins": 1.1445512771606445,
+      "rewards/rejected": -3.2576184272766113,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 42.82234513435798,
+      "learning_rate": 1.4920970783889737e-08,
+      "logits/chosen": -1.784874677658081,
+      "logits/rejected": -1.022477388381958,
+      "logps/chosen": -403.08294677734375,
+      "logps/rejected": -465.8329162597656,
+      "loss": 0.4906,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.028446912765503,
+      "rewards/margins": 0.7884195446968079,
+      "rewards/rejected": -2.816866636276245,
+      "step": 1720
+    },
+    {
+      "epoch": 0.9055221146296781,
+      "grad_norm": 67.83919792587483,
+      "learning_rate": 1.340549934783164e-08,
+      "logits/chosen": -1.6942017078399658,
+      "logits/rejected": -1.3202614784240723,
+      "logps/chosen": -391.9894104003906,
+      "logps/rejected": -499.2957458496094,
+      "loss": 0.5082,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1304984092712402,
+      "rewards/margins": 0.9605563879013062,
+      "rewards/rejected": -3.091054916381836,
+      "step": 1730
+    },
+    {
+      "epoch": 0.9107563465061502,
+      "grad_norm": 38.38762358526929,
+      "learning_rate": 1.1969048553059608e-08,
+      "logits/chosen": -1.5242860317230225,
+      "logits/rejected": -1.0694557428359985,
+      "logps/chosen": -385.64447021484375,
+      "logps/rejected": -532.7508544921875,
+      "loss": 0.5175,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.149869441986084,
+      "rewards/margins": 1.1629979610443115,
+      "rewards/rejected": -3.3128669261932373,
+      "step": 1740
+    },
+    {
+      "epoch": 0.9159905783826223,
+      "grad_norm": 47.57788484474261,
+      "learning_rate": 1.06120981624703e-08,
+      "logits/chosen": -1.8843555450439453,
+      "logits/rejected": -0.6692565083503723,
+      "logps/chosen": -416.1067810058594,
+      "logps/rejected": -449.87103271484375,
+      "loss": 0.524,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.03684401512146,
+      "rewards/margins": 0.8309102058410645,
+      "rewards/rejected": -2.8677542209625244,
+      "step": 1750
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 51.79734307723889,
+      "learning_rate": 9.335101386471284e-09,
+      "logits/chosen": -1.769464135169983,
+      "logits/rejected": -1.5171881914138794,
+      "logps/chosen": -366.7747497558594,
+      "logps/rejected": -517.2105712890625,
+      "loss": 0.4596,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.8451915979385376,
+      "rewards/margins": 1.2242134809494019,
+      "rewards/rejected": -3.0694050788879395,
+      "step": 1760
+    },
+    {
+      "epoch": 0.9264590421355666,
+      "grad_norm": 59.243332805566055,
+      "learning_rate": 8.138484731612273e-09,
+      "logits/chosen": -1.610945701599121,
+      "logits/rejected": -0.8597782850265503,
+      "logps/chosen": -412.3548278808594,
+      "logps/rejected": -507.70050048828125,
+      "loss": 0.4844,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.1251347064971924,
+      "rewards/margins": 1.0798755884170532,
+      "rewards/rejected": -3.205010175704956,
+      "step": 1770
+    },
+    {
+      "epoch": 0.9316932740120387,
+      "grad_norm": 35.23329301018955,
+      "learning_rate": 7.0226478581355e-09,
+      "logits/chosen": -1.5735182762145996,
+      "logits/rejected": -1.0823094844818115,
+      "logps/chosen": -382.31378173828125,
+      "logps/rejected": -459.026611328125,
+      "loss": 0.522,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.1322665214538574,
+      "rewards/margins": 0.7317706346511841,
+      "rewards/rejected": -2.86403751373291,
+      "step": 1780
+    },
+    {
+      "epoch": 0.9369275058885108,
+      "grad_norm": 39.12208203761793,
+      "learning_rate": 5.987963446492383e-09,
+      "logits/chosen": -1.6163190603256226,
+      "logits/rejected": -1.3726081848144531,
+      "logps/chosen": -368.1934509277344,
+      "logps/rejected": -484.42620849609375,
+      "loss": 0.4783,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.939570426940918,
+      "rewards/margins": 0.9698532819747925,
+      "rewards/rejected": -2.909423828125,
+      "step": 1790
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 45.32578662406385,
+      "learning_rate": 5.0347770728713935e-09,
+      "logits/chosen": -1.5549806356430054,
+      "logits/rejected": -0.9020807147026062,
+      "logps/chosen": -382.2792053222656,
+      "logps/rejected": -480.25714111328125,
+      "loss": 0.4825,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.1894912719726562,
+      "rewards/margins": 0.8568480610847473,
+      "rewards/rejected": -3.046339273452759,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9473959696414551,
+      "grad_norm": 51.50392990407485,
+      "learning_rate": 4.1634070937782424e-09,
+      "logits/chosen": -1.563454270362854,
+      "logits/rejected": -1.4620710611343384,
+      "logps/chosen": -374.35662841796875,
+      "logps/rejected": -459.2840881347656,
+      "loss": 0.4949,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0908713340759277,
+      "rewards/margins": 0.8073171377182007,
+      "rewards/rejected": -2.898188829421997,
+      "step": 1810
+    },
+    {
+      "epoch": 0.9526302015179272,
+      "grad_norm": 52.57515906506144,
+      "learning_rate": 3.3741445397075797e-09,
+      "logits/chosen": -1.5654033422470093,
+      "logits/rejected": -0.9612579345703125,
+      "logps/chosen": -403.11767578125,
+      "logps/rejected": -518.7571411132812,
+      "loss": 0.499,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.0279488563537598,
+      "rewards/margins": 1.2528560161590576,
+      "rewards/rejected": -3.2808051109313965,
+      "step": 1820
+    },
+    {
+      "epoch": 0.9578644333943994,
+      "grad_norm": 80.78348799446286,
+      "learning_rate": 2.667253017941018e-09,
+      "logits/chosen": -1.5542982816696167,
+      "logits/rejected": -1.2357369661331177,
+      "logps/chosen": -414.04571533203125,
+      "logps/rejected": -515.9486083984375,
+      "loss": 0.5405,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.095529079437256,
+      "rewards/margins": 0.8272086977958679,
+      "rewards/rejected": -2.9227375984191895,
+      "step": 1830
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 60.481486866817654,
+      "learning_rate": 2.0429686245045097e-09,
+      "logits/chosen": -1.7054321765899658,
+      "logits/rejected": -0.9457400441169739,
+      "logps/chosen": -383.46612548828125,
+      "logps/rejected": -498.7964782714844,
+      "loss": 0.4679,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.065452814102173,
+      "rewards/margins": 1.179323434829712,
+      "rewards/rejected": -3.244776487350464,
+      "step": 1840
+    },
+    {
+      "epoch": 0.9683328971473436,
+      "grad_norm": 42.47290413747811,
+      "learning_rate": 1.5014998653141708e-09,
+      "logits/chosen": -1.4126255512237549,
+      "logits/rejected": -1.1500160694122314,
+      "logps/chosen": -424.77337646484375,
+      "logps/rejected": -545.006591796875,
+      "loss": 0.4857,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.2447969913482666,
+      "rewards/margins": 1.013723611831665,
+      "rewards/rejected": -3.2585208415985107,
+      "step": 1850
+    },
+    {
+      "epoch": 0.9735671290238157,
+      "grad_norm": 55.24460892129673,
+      "learning_rate": 1.0430275865371263e-09,
+      "logits/chosen": -1.460265874862671,
+      "logits/rejected": -0.5893803238868713,
+      "logps/chosen": -364.1796569824219,
+      "logps/rejected": -458.278564453125,
+      "loss": 0.4837,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.0408921241760254,
+      "rewards/margins": 1.031553864479065,
+      "rewards/rejected": -3.0724456310272217,
+      "step": 1860
+    },
+    {
+      "epoch": 0.9788013609002879,
+      "grad_norm": 78.57983765704049,
+      "learning_rate": 6.677049141901314e-10,
+      "logits/chosen": -1.6051515340805054,
+      "logits/rejected": -0.9829438924789429,
+      "logps/chosen": -398.05267333984375,
+      "logps/rejected": -477.8648376464844,
+      "loss": 0.4942,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.0092108249664307,
+      "rewards/margins": 0.8877508044242859,
+      "rewards/rejected": -2.8969614505767822,
+      "step": 1870
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 37.16308130632785,
+      "learning_rate": 3.7565720299687077e-10,
+      "logits/chosen": -1.8912632465362549,
+      "logits/rejected": -0.9422454833984375,
+      "logps/chosen": -400.77197265625,
+      "logps/rejected": -511.6893005371094,
+      "loss": 0.4815,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.160900115966797,
+      "rewards/margins": 1.268087387084961,
+      "rewards/rejected": -3.428987503051758,
+      "step": 1880
+    },
+    {
+      "epoch": 0.9892698246532321,
+      "grad_norm": 54.378639302860876,
+      "learning_rate": 1.6698199452053197e-10,
+      "logits/chosen": -1.7411140203475952,
+      "logits/rejected": -1.1444064378738403,
+      "logps/chosen": -405.96466064453125,
+      "logps/rejected": -516.4884643554688,
+      "loss": 0.5036,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.084994077682495,
+      "rewards/margins": 0.9552907943725586,
+      "rewards/rejected": -3.0402846336364746,
+      "step": 1890
+    },
+    {
+      "epoch": 0.9945040565297043,
+      "grad_norm": 55.60143338165864,
+      "learning_rate": 4.174898458556009e-11,
+      "logits/chosen": -1.7632405757904053,
+      "logits/rejected": -1.2880979776382446,
+      "logps/chosen": -400.0120544433594,
+      "logps/rejected": -445.19036865234375,
+      "loss": 0.516,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.067795515060425,
+      "rewards/margins": 0.6980506777763367,
+      "rewards/rejected": -2.7658464908599854,
+      "step": 1900
+    },
+    {
+      "epoch": 0.9997382884061764,
+      "grad_norm": 50.31539314937899,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.4952467679977417,
+      "logits/rejected": -0.9173043966293335,
+      "logps/chosen": -421.25,
+      "logps/rejected": -543.969482421875,
+      "loss": 0.4891,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.100468635559082,
+      "rewards/margins": 1.1773890256881714,
+      "rewards/rejected": -3.277857542037964,
+      "step": 1910
+    },
+    {
+      "epoch": 0.9997382884061764,
+      "step": 1910,
+      "total_flos": 0.0,
+      "train_loss": 0.5374546366836388,
+      "train_runtime": 38043.1849,
+      "train_samples_per_second": 1.607,
+      "train_steps_per_second": 0.05
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1910,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}