{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.29041064064587324,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0005808212812917465,
      "grad_norm": 306.3320007324219,
      "learning_rate": 4.998547356188263e-06,
      "logits/chosen": -0.7514113187789917,
      "logits/rejected": -0.6686298251152039,
      "logps/chosen": -75.72093200683594,
      "logps/rejected": -73.8106918334961,
      "loss": 13.8629,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.001161642562583493,
      "grad_norm": 301.2439270019531,
      "learning_rate": 4.997094712376526e-06,
      "logits/chosen": -0.794822096824646,
      "logits/rejected": -0.7371929287910461,
      "logps/chosen": -72.30989074707031,
      "logps/rejected": -67.51399993896484,
      "loss": 13.9577,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.0054689692333340645,
      "rewards/margins": -0.009410643950104713,
      "rewards/rejected": 0.003941674716770649,
      "step": 2
    },
    {
      "epoch": 0.0017424638438752395,
      "grad_norm": 296.5701904296875,
      "learning_rate": 4.995642068564789e-06,
      "logits/chosen": -0.8363990783691406,
      "logits/rejected": -0.8187875747680664,
      "logps/chosen": -71.92262268066406,
      "logps/rejected": -72.27050018310547,
      "loss": 13.7847,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0027333260513842106,
      "rewards/margins": 0.008102846331894398,
      "rewards/rejected": -0.0053695198148489,
      "step": 3
    },
    {
      "epoch": 0.002323285125166986,
      "grad_norm": 345.2494201660156,
      "learning_rate": 4.9941894247530506e-06,
      "logits/chosen": -0.7175111174583435,
      "logits/rejected": -0.7101837396621704,
      "logps/chosen": -77.45024108886719,
      "logps/rejected": -76.15581512451172,
      "loss": 13.9709,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01066683791577816,
      "rewards/margins": -0.01057196594774723,
      "rewards/rejected": -9.487159695709124e-05,
      "step": 4
    },
    {
      "epoch": 0.0029041064064587326,
      "grad_norm": 295.31768798828125,
      "learning_rate": 4.992736780941313e-06,
      "logits/chosen": -0.7365175485610962,
      "logits/rejected": -0.7447739839553833,
      "logps/chosen": -76.33888244628906,
      "logps/rejected": -61.07477951049805,
      "loss": 13.8975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0010636046063154936,
      "rewards/margins": -0.0031540922354906797,
      "rewards/rejected": 0.004217695910483599,
      "step": 5
    },
    {
      "epoch": 0.003484927687750479,
      "grad_norm": 312.8099060058594,
      "learning_rate": 4.991284137129576e-06,
      "logits/chosen": -0.875682532787323,
      "logits/rejected": -0.8198660016059875,
      "logps/chosen": -79.96182250976562,
      "logps/rejected": -77.87804412841797,
      "loss": 13.6993,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01062581967562437,
      "rewards/margins": 0.016683798283338547,
      "rewards/rejected": -0.0060579776763916016,
      "step": 6
    },
    {
      "epoch": 0.004065748969042226,
      "grad_norm": 317.39288330078125,
      "learning_rate": 4.989831493317839e-06,
      "logits/chosen": -0.6050316095352173,
      "logits/rejected": -0.6816262602806091,
      "logps/chosen": -70.26258850097656,
      "logps/rejected": -75.84834289550781,
      "loss": 13.8254,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010981644503772259,
      "rewards/margins": 0.003875770838931203,
      "rewards/rejected": 0.007105874828994274,
      "step": 7
    },
    {
      "epoch": 0.004646570250333972,
      "grad_norm": 321.52996826171875,
      "learning_rate": 4.9883788495061015e-06,
      "logits/chosen": -0.8232254981994629,
      "logits/rejected": -0.7795180082321167,
      "logps/chosen": -72.38011169433594,
      "logps/rejected": -67.78025817871094,
      "loss": 13.6941,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.017304277047514915,
      "rewards/margins": 0.01702544279396534,
      "rewards/rejected": 0.00027883489383384585,
      "step": 8
    },
    {
      "epoch": 0.005227391531625719,
      "grad_norm": 311.1319885253906,
      "learning_rate": 4.986926205694364e-06,
      "logits/chosen": -0.7957251667976379,
      "logits/rejected": -0.748576819896698,
      "logps/chosen": -76.44227600097656,
      "logps/rejected": -72.32237243652344,
      "loss": 13.8266,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007577553391456604,
      "rewards/margins": 0.003815555479377508,
      "rewards/rejected": 0.0037619969807565212,
      "step": 9
    },
    {
      "epoch": 0.005808212812917465,
      "grad_norm": 290.04388427734375,
      "learning_rate": 4.985473561882627e-06,
      "logits/chosen": -0.851279079914093,
      "logits/rejected": -0.8175627589225769,
      "logps/chosen": -62.6363639831543,
      "logps/rejected": -66.80535125732422,
      "loss": 13.8568,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001977672567591071,
      "rewards/margins": 0.0007607266306877136,
      "rewards/rejected": 0.0012169458204880357,
      "step": 10
    },
    {
      "epoch": 0.006389034094209212,
      "grad_norm": 477.3677673339844,
      "learning_rate": 4.984020918070889e-06,
      "logits/chosen": -0.7694743871688843,
      "logits/rejected": -0.7370525598526001,
      "logps/chosen": -68.05220794677734,
      "logps/rejected": -73.65959167480469,
      "loss": 13.9053,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0020100977271795273,
      "rewards/margins": -0.004024811089038849,
      "rewards/rejected": 0.002014713129028678,
      "step": 11
    },
    {
      "epoch": 0.006969855375500958,
      "grad_norm": 339.96746826171875,
      "learning_rate": 4.982568274259152e-06,
      "logits/chosen": -0.4584922194480896,
      "logits/rejected": -0.4653104245662689,
      "logps/chosen": -68.20719146728516,
      "logps/rejected": -76.61314392089844,
      "loss": 13.789,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.010751953348517418,
      "rewards/margins": 0.007597408257424831,
      "rewards/rejected": 0.003154544625431299,
      "step": 12
    },
    {
      "epoch": 0.0075506766567927045,
      "grad_norm": 307.5588684082031,
      "learning_rate": 4.9811156304474144e-06,
      "logits/chosen": -0.6956412196159363,
      "logits/rejected": -0.5891402959823608,
      "logps/chosen": -66.07670593261719,
      "logps/rejected": -77.52650451660156,
      "loss": 13.8012,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0009560681646689773,
      "rewards/margins": 0.006629952695220709,
      "rewards/rejected": -0.0075860219076275826,
      "step": 13
    },
    {
      "epoch": 0.008131497938084452,
      "grad_norm": 324.5359802246094,
      "learning_rate": 4.979662986635677e-06,
      "logits/chosen": -0.5961264371871948,
      "logits/rejected": -0.6395691633224487,
      "logps/chosen": -76.06859588623047,
      "logps/rejected": -75.65780639648438,
      "loss": 13.8604,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0018664164235815406,
      "rewards/margins": 0.0006155198207125068,
      "rewards/rejected": -0.0024819376412779093,
      "step": 14
    },
    {
      "epoch": 0.008712319219376197,
      "grad_norm": 321.2744445800781,
      "learning_rate": 4.97821034282394e-06,
      "logits/chosen": -0.609241247177124,
      "logits/rejected": -0.6541947722434998,
      "logps/chosen": -72.74789428710938,
      "logps/rejected": -78.14617156982422,
      "loss": 13.6613,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.007168769836425781,
      "rewards/margins": 0.020653218030929565,
      "rewards/rejected": -0.013484450057148933,
      "step": 15
    },
    {
      "epoch": 0.009293140500667945,
      "grad_norm": 321.91412353515625,
      "learning_rate": 4.976757699012203e-06,
      "logits/chosen": -0.8816198110580444,
      "logits/rejected": -1.0502017736434937,
      "logps/chosen": -81.1220474243164,
      "logps/rejected": -78.71932220458984,
      "loss": 14.1071,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.012946806848049164,
      "rewards/margins": -0.023911792784929276,
      "rewards/rejected": 0.010964984074234962,
      "step": 16
    },
    {
      "epoch": 0.00987396178195969,
      "grad_norm": 285.6275939941406,
      "learning_rate": 4.9753050552004654e-06,
      "logits/chosen": -0.7112148404121399,
      "logits/rejected": -0.6412473917007446,
      "logps/chosen": -65.52027893066406,
      "logps/rejected": -71.49274444580078,
      "loss": 13.7638,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.007086105644702911,
      "rewards/margins": 0.010165892541408539,
      "rewards/rejected": -0.00307978643104434,
      "step": 17
    },
    {
      "epoch": 0.010454783063251438,
      "grad_norm": 328.0557556152344,
      "learning_rate": 4.973852411388727e-06,
      "logits/chosen": -0.756773829460144,
      "logits/rejected": -0.853185772895813,
      "logps/chosen": -71.16859436035156,
      "logps/rejected": -70.41301727294922,
      "loss": 13.7103,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.006009445525705814,
      "rewards/margins": 0.016000624746084213,
      "rewards/rejected": -0.022010069340467453,
      "step": 18
    },
    {
      "epoch": 0.011035604344543185,
      "grad_norm": 286.8959045410156,
      "learning_rate": 4.97239976757699e-06,
      "logits/chosen": -0.7466567754745483,
      "logits/rejected": -0.8737386465072632,
      "logps/chosen": -70.04942321777344,
      "logps/rejected": -69.66856384277344,
      "loss": 13.9803,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.00499460194259882,
      "rewards/margins": -0.011114511638879776,
      "rewards/rejected": 0.006119909696280956,
      "step": 19
    },
    {
      "epoch": 0.01161642562583493,
      "grad_norm": 322.2734069824219,
      "learning_rate": 4.970947123765253e-06,
      "logits/chosen": -0.7975467443466187,
      "logits/rejected": -0.9792510271072388,
      "logps/chosen": -76.38298034667969,
      "logps/rejected": -82.6656265258789,
      "loss": 13.9642,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.015522710978984833,
      "rewards/margins": -0.009529724717140198,
      "rewards/rejected": -0.005992984864860773,
      "step": 20
    },
    {
      "epoch": 0.012197246907126678,
      "grad_norm": 316.05517578125,
      "learning_rate": 4.9694944799535164e-06,
      "logits/chosen": -0.9200956225395203,
      "logits/rejected": -0.8602321743965149,
      "logps/chosen": -69.63390350341797,
      "logps/rejected": -80.1779556274414,
      "loss": 13.8359,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017169209197163582,
      "rewards/margins": 0.004035423509776592,
      "rewards/rejected": -0.0212046317756176,
      "step": 21
    },
    {
      "epoch": 0.012778068188418423,
      "grad_norm": 335.1014404296875,
      "learning_rate": 4.968041836141778e-06,
      "logits/chosen": -0.787733256816864,
      "logits/rejected": -0.7936286926269531,
      "logps/chosen": -79.13917541503906,
      "logps/rejected": -66.25904083251953,
      "loss": 14.2123,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.030913371592760086,
      "rewards/margins": -0.03371546417474747,
      "rewards/rejected": 0.0028020956087857485,
      "step": 22
    },
    {
      "epoch": 0.01335888946971017,
      "grad_norm": 294.2160339355469,
      "learning_rate": 4.966589192330041e-06,
      "logits/chosen": -0.6786571741104126,
      "logits/rejected": -0.7551315426826477,
      "logps/chosen": -69.29032897949219,
      "logps/rejected": -69.82914733886719,
      "loss": 13.8587,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.006731729954481125,
      "rewards/margins": 0.0010073954472318292,
      "rewards/rejected": -0.00773912388831377,
      "step": 23
    },
    {
      "epoch": 0.013939710751001916,
      "grad_norm": 323.32000732421875,
      "learning_rate": 4.965136548518304e-06,
      "logits/chosen": -0.9059945940971375,
      "logits/rejected": -0.7469512224197388,
      "logps/chosen": -78.02278137207031,
      "logps/rejected": -66.72486877441406,
      "loss": 13.9736,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.030265387147665024,
      "rewards/margins": -0.009611329063773155,
      "rewards/rejected": -0.02065405808389187,
      "step": 24
    },
    {
      "epoch": 0.014520532032293663,
      "grad_norm": 333.2950744628906,
      "learning_rate": 4.963683904706567e-06,
      "logits/chosen": -0.8123146891593933,
      "logits/rejected": -0.7186424136161804,
      "logps/chosen": -76.54938507080078,
      "logps/rejected": -67.63246154785156,
      "loss": 13.8885,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.013149453327059746,
      "rewards/margins": -0.0020573907531797886,
      "rewards/rejected": -0.01109206210821867,
      "step": 25
    },
    {
      "epoch": 0.015101353313585409,
      "grad_norm": 357.38775634765625,
      "learning_rate": 4.962231260894829e-06,
      "logits/chosen": -0.8334075808525085,
      "logits/rejected": -0.8764799237251282,
      "logps/chosen": -77.63631439208984,
      "logps/rejected": -77.82304382324219,
      "loss": 14.058,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.015404301695525646,
      "rewards/margins": -0.01881382428109646,
      "rewards/rejected": 0.0034095190931111574,
      "step": 26
    },
    {
      "epoch": 0.015682174594877155,
      "grad_norm": 422.6893615722656,
      "learning_rate": 4.960778617083092e-06,
      "logits/chosen": -0.9417294263839722,
      "logits/rejected": -1.0054762363433838,
      "logps/chosen": -73.45500183105469,
      "logps/rejected": -71.62086486816406,
      "loss": 13.9493,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02026461809873581,
      "rewards/margins": -0.007518300320953131,
      "rewards/rejected": -0.012746316380798817,
      "step": 27
    },
    {
      "epoch": 0.016262995876168904,
      "grad_norm": 302.3515625,
      "learning_rate": 4.959325973271355e-06,
      "logits/chosen": -0.7729172110557556,
      "logits/rejected": -0.9286600947380066,
      "logps/chosen": -74.0387191772461,
      "logps/rejected": -74.87562561035156,
      "loss": 13.7489,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.001342860283330083,
      "rewards/margins": 0.012399435043334961,
      "rewards/rejected": -0.0137422950938344,
      "step": 28
    },
    {
      "epoch": 0.01684381715746065,
      "grad_norm": 311.303466796875,
      "learning_rate": 4.957873329459617e-06,
      "logits/chosen": -0.7707468271255493,
      "logits/rejected": -0.7816058993339539,
      "logps/chosen": -80.28218078613281,
      "logps/rejected": -76.27729034423828,
      "loss": 13.7301,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.017700577154755592,
      "rewards/margins": 0.01469388883560896,
      "rewards/rejected": -0.03239446505904198,
      "step": 29
    },
    {
      "epoch": 0.017424638438752395,
      "grad_norm": 272.25091552734375,
      "learning_rate": 4.9564206856478795e-06,
      "logits/chosen": -0.9470396041870117,
      "logits/rejected": -1.0160866975784302,
      "logps/chosen": -63.82170867919922,
      "logps/rejected": -67.61204528808594,
      "loss": 13.7505,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0032949731685221195,
      "rewards/margins": 0.011799006722867489,
      "rewards/rejected": -0.008504033088684082,
      "step": 30
    },
    {
      "epoch": 0.018005459720044144,
      "grad_norm": 320.925048828125,
      "learning_rate": 4.954968041836142e-06,
      "logits/chosen": -0.8708783388137817,
      "logits/rejected": -0.8632427453994751,
      "logps/chosen": -73.67878723144531,
      "logps/rejected": -74.83086395263672,
      "loss": 13.6099,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002552690450102091,
      "rewards/margins": 0.026570502668619156,
      "rewards/rejected": -0.02912319265305996,
      "step": 31
    },
    {
      "epoch": 0.01858628100133589,
      "grad_norm": 303.17852783203125,
      "learning_rate": 4.953515398024405e-06,
      "logits/chosen": -0.7567359805107117,
      "logits/rejected": -0.8445581197738647,
      "logps/chosen": -66.88877868652344,
      "logps/rejected": -71.22685241699219,
      "loss": 13.845,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01957935281097889,
      "rewards/margins": 0.0028636164497584105,
      "rewards/rejected": -0.022442970424890518,
      "step": 32
    },
    {
      "epoch": 0.019167102282627635,
      "grad_norm": 317.728515625,
      "learning_rate": 4.952062754212668e-06,
      "logits/chosen": -0.917065441608429,
      "logits/rejected": -0.8019243478775024,
      "logps/chosen": -72.54550170898438,
      "logps/rejected": -81.64886474609375,
      "loss": 13.6166,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.011156129650771618,
      "rewards/margins": 0.025838393718004227,
      "rewards/rejected": -0.03699452430009842,
      "step": 33
    },
    {
      "epoch": 0.01974792356391938,
      "grad_norm": 295.3751525878906,
      "learning_rate": 4.9506101104009305e-06,
      "logits/chosen": -0.8145904541015625,
      "logits/rejected": -0.7906870245933533,
      "logps/chosen": -64.89479064941406,
      "logps/rejected": -76.6470947265625,
      "loss": 13.7107,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.01140755694359541,
      "rewards/margins": 0.017068836838006973,
      "rewards/rejected": -0.02847639098763466,
      "step": 34
    },
    {
      "epoch": 0.02032874484521113,
      "grad_norm": 305.5505676269531,
      "learning_rate": 4.949157466589193e-06,
      "logits/chosen": -0.7340711355209351,
      "logits/rejected": -0.7537750005722046,
      "logps/chosen": -75.50181579589844,
      "logps/rejected": -70.16544342041016,
      "loss": 14.1136,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.028647294268012047,
      "rewards/margins": -0.023600969463586807,
      "rewards/rejected": -0.005046320613473654,
      "step": 35
    },
    {
      "epoch": 0.020909566126502875,
      "grad_norm": 315.96038818359375,
      "learning_rate": 4.947704822777455e-06,
      "logits/chosen": -0.8396091461181641,
      "logits/rejected": -0.9497518539428711,
      "logps/chosen": -72.89092254638672,
      "logps/rejected": -83.45375061035156,
      "loss": 13.8082,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02720179595053196,
      "rewards/margins": 0.006764040794223547,
      "rewards/rejected": -0.033965837210416794,
      "step": 36
    },
    {
      "epoch": 0.02149038740779462,
      "grad_norm": 320.8994445800781,
      "learning_rate": 4.946252178965718e-06,
      "logits/chosen": -0.7043382525444031,
      "logits/rejected": -0.8497918248176575,
      "logps/chosen": -80.10662841796875,
      "logps/rejected": -80.18870544433594,
      "loss": 13.8802,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04296343773603439,
      "rewards/margins": 0.0006133742863312364,
      "rewards/rejected": -0.043576814234256744,
      "step": 37
    },
    {
      "epoch": 0.02207120868908637,
      "grad_norm": 323.31488037109375,
      "learning_rate": 4.944799535153981e-06,
      "logits/chosen": -0.9256412386894226,
      "logits/rejected": -0.9272140264511108,
      "logps/chosen": -79.0914306640625,
      "logps/rejected": -79.18199157714844,
      "loss": 13.6203,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.016189271584153175,
      "rewards/margins": 0.02659023180603981,
      "rewards/rejected": -0.042779501527547836,
      "step": 38
    },
    {
      "epoch": 0.022652029970378115,
      "grad_norm": 309.1510314941406,
      "learning_rate": 4.943346891342243e-06,
      "logits/chosen": -0.7942522168159485,
      "logits/rejected": -0.9100838899612427,
      "logps/chosen": -78.5674057006836,
      "logps/rejected": -69.88294982910156,
      "loss": 13.854,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.028528030961751938,
      "rewards/margins": 0.0029734233394265175,
      "rewards/rejected": -0.03150145336985588,
      "step": 39
    },
    {
      "epoch": 0.02323285125166986,
      "grad_norm": 326.2637023925781,
      "learning_rate": 4.941894247530506e-06,
      "logits/chosen": -0.8673677444458008,
      "logits/rejected": -0.8385285139083862,
      "logps/chosen": -85.15605926513672,
      "logps/rejected": -84.39887237548828,
      "loss": 13.801,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04420323297381401,
      "rewards/margins": 0.006766452454030514,
      "rewards/rejected": -0.0509696826338768,
      "step": 40
    },
    {
      "epoch": 0.023813672532961606,
      "grad_norm": 309.16204833984375,
      "learning_rate": 4.940441603718769e-06,
      "logits/chosen": -0.8228281736373901,
      "logits/rejected": -0.8817359209060669,
      "logps/chosen": -76.8722915649414,
      "logps/rejected": -74.45531463623047,
      "loss": 13.6428,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009386795572936535,
      "rewards/margins": 0.024640636518597603,
      "rewards/rejected": -0.03402743488550186,
      "step": 41
    },
    {
      "epoch": 0.024394493814253355,
      "grad_norm": 322.296630859375,
      "learning_rate": 4.938988959907032e-06,
      "logits/chosen": -0.7055200338363647,
      "logits/rejected": -0.779880166053772,
      "logps/chosen": -75.47132873535156,
      "logps/rejected": -75.7305908203125,
      "loss": 13.6296,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.015317773446440697,
      "rewards/margins": 0.025457333773374557,
      "rewards/rejected": -0.0407751128077507,
      "step": 42
    },
    {
      "epoch": 0.0249753150955451,
      "grad_norm": 334.68182373046875,
      "learning_rate": 4.9375363160952935e-06,
      "logits/chosen": -0.7710026502609253,
      "logits/rejected": -0.82500159740448,
      "logps/chosen": -77.85375213623047,
      "logps/rejected": -74.20629119873047,
      "loss": 13.6993,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.007363433483988047,
      "rewards/margins": 0.018297644332051277,
      "rewards/rejected": -0.025661081075668335,
      "step": 43
    },
    {
      "epoch": 0.025556136376836847,
      "grad_norm": 317.21905517578125,
      "learning_rate": 4.936083672283556e-06,
      "logits/chosen": -0.988287627696991,
      "logits/rejected": -0.9961759448051453,
      "logps/chosen": -77.48571014404297,
      "logps/rejected": -74.44812774658203,
      "loss": 14.0204,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04125748947262764,
      "rewards/margins": -0.01405995525419712,
      "rewards/rejected": -0.02719753421843052,
      "step": 44
    },
    {
      "epoch": 0.026136957658128592,
      "grad_norm": 315.16363525390625,
      "learning_rate": 4.934631028471819e-06,
      "logits/chosen": -0.8442651629447937,
      "logits/rejected": -0.9505764842033386,
      "logps/chosen": -74.02120208740234,
      "logps/rejected": -72.55667114257812,
      "loss": 13.8445,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.009219218045473099,
      "rewards/margins": 0.003767718095332384,
      "rewards/rejected": -0.01298693846911192,
      "step": 45
    },
    {
      "epoch": 0.02671777893942034,
      "grad_norm": 333.017578125,
      "learning_rate": 4.933178384660082e-06,
      "logits/chosen": -0.8707895278930664,
      "logits/rejected": -0.8848034739494324,
      "logps/chosen": -82.85108947753906,
      "logps/rejected": -82.15937805175781,
      "loss": 14.1048,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.05764341354370117,
      "rewards/margins": -0.019157161936163902,
      "rewards/rejected": -0.03848625347018242,
      "step": 46
    },
    {
      "epoch": 0.027298600220712087,
      "grad_norm": 328.5982971191406,
      "learning_rate": 4.9317257408483445e-06,
      "logits/chosen": -0.8642932176589966,
      "logits/rejected": -0.8195087313652039,
      "logps/chosen": -75.42310333251953,
      "logps/rejected": -78.54690551757812,
      "loss": 13.8605,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.027368813753128052,
      "rewards/margins": 0.0027305304538458586,
      "rewards/rejected": -0.03009933792054653,
      "step": 47
    },
    {
      "epoch": 0.027879421502003832,
      "grad_norm": 332.7618713378906,
      "learning_rate": 4.930273097036607e-06,
      "logits/chosen": -0.8657892346382141,
      "logits/rejected": -0.9026430249214172,
      "logps/chosen": -86.77098083496094,
      "logps/rejected": -70.10636901855469,
      "loss": 13.9202,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03694070875644684,
      "rewards/margins": -0.0033356398344039917,
      "rewards/rejected": -0.033605072647333145,
      "step": 48
    },
    {
      "epoch": 0.02846024278329558,
      "grad_norm": 326.447265625,
      "learning_rate": 4.928820453224869e-06,
      "logits/chosen": -0.931209921836853,
      "logits/rejected": -0.7586521506309509,
      "logps/chosen": -78.26404571533203,
      "logps/rejected": -75.22278594970703,
      "loss": 13.6775,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.009883576072752476,
      "rewards/margins": 0.02028750441968441,
      "rewards/rejected": -0.030171077698469162,
      "step": 49
    },
    {
      "epoch": 0.029041064064587327,
      "grad_norm": 323.82501220703125,
      "learning_rate": 4.927367809413132e-06,
      "logits/chosen": -0.8577003479003906,
      "logits/rejected": -0.9445897340774536,
      "logps/chosen": -80.17283630371094,
      "logps/rejected": -73.15702819824219,
      "loss": 13.8828,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04805426672101021,
      "rewards/margins": 6.009042408550158e-05,
      "rewards/rejected": -0.0481143593788147,
      "step": 50
    },
    {
      "epoch": 0.029621885345879072,
      "grad_norm": 345.8555603027344,
      "learning_rate": 4.925915165601395e-06,
      "logits/chosen": -0.8427948951721191,
      "logits/rejected": -0.6843789219856262,
      "logps/chosen": -67.90852355957031,
      "logps/rejected": -78.57261657714844,
      "loss": 13.9293,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03557172790169716,
      "rewards/margins": -0.005063153803348541,
      "rewards/rejected": -0.030508574098348618,
      "step": 51
    },
    {
      "epoch": 0.030202706627170818,
      "grad_norm": 330.11419677734375,
      "learning_rate": 4.924462521789657e-06,
      "logits/chosen": -0.8312528729438782,
      "logits/rejected": -0.8623727560043335,
      "logps/chosen": -73.96318054199219,
      "logps/rejected": -74.79844665527344,
      "loss": 14.0308,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.040969304740428925,
      "rewards/margins": -0.014493905007839203,
      "rewards/rejected": -0.02647540345788002,
      "step": 52
    },
    {
      "epoch": 0.030783527908462567,
      "grad_norm": 439.39111328125,
      "learning_rate": 4.92300987797792e-06,
      "logits/chosen": -0.8176994323730469,
      "logits/rejected": -0.7079430818557739,
      "logps/chosen": -72.11351013183594,
      "logps/rejected": -75.1050033569336,
      "loss": 13.9598,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.011684712953865528,
      "rewards/margins": -0.007551603019237518,
      "rewards/rejected": -0.004133109003305435,
      "step": 53
    },
    {
      "epoch": 0.03136434918975431,
      "grad_norm": 299.9485778808594,
      "learning_rate": 4.921557234166183e-06,
      "logits/chosen": -0.7416559457778931,
      "logits/rejected": -0.7767287492752075,
      "logps/chosen": -70.76741790771484,
      "logps/rejected": -73.53133392333984,
      "loss": 13.7678,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0056773098185658455,
      "rewards/margins": 0.010591240599751472,
      "rewards/rejected": -0.016268549486994743,
      "step": 54
    },
    {
      "epoch": 0.03194517047104606,
      "grad_norm": 294.43988037109375,
      "learning_rate": 4.920104590354446e-06,
      "logits/chosen": -0.6667272448539734,
      "logits/rejected": -0.8636151552200317,
      "logps/chosen": -63.9805793762207,
      "logps/rejected": -74.08900451660156,
      "loss": 13.9136,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.028553063049912453,
      "rewards/margins": -0.0023740821052342653,
      "rewards/rejected": -0.026178985834121704,
      "step": 55
    },
    {
      "epoch": 0.03252599175233781,
      "grad_norm": 315.2623596191406,
      "learning_rate": 4.9186519465427075e-06,
      "logits/chosen": -0.9049865007400513,
      "logits/rejected": -0.9067096710205078,
      "logps/chosen": -71.81194305419922,
      "logps/rejected": -69.87062072753906,
      "loss": 13.6316,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.010484697297215462,
      "rewards/margins": 0.025315653532743454,
      "rewards/rejected": -0.03580035641789436,
      "step": 56
    },
    {
      "epoch": 0.03310681303362955,
      "grad_norm": 317.02972412109375,
      "learning_rate": 4.91719930273097e-06,
      "logits/chosen": -0.8822166323661804,
      "logits/rejected": -0.8712530136108398,
      "logps/chosen": -73.8038330078125,
      "logps/rejected": -70.23847198486328,
      "loss": 13.7197,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0062989904545247555,
      "rewards/margins": 0.016949044540524483,
      "rewards/rejected": -0.010650052689015865,
      "step": 57
    },
    {
      "epoch": 0.0336876343149213,
      "grad_norm": 379.47076416015625,
      "learning_rate": 4.915746658919233e-06,
      "logits/chosen": -0.9493061304092407,
      "logits/rejected": -0.8905242681503296,
      "logps/chosen": -77.11824798583984,
      "logps/rejected": -74.96271514892578,
      "loss": 13.7691,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.024838361889123917,
      "rewards/margins": 0.011607111431658268,
      "rewards/rejected": -0.03644547611474991,
      "step": 58
    },
    {
      "epoch": 0.03426845559621305,
      "grad_norm": 306.8603210449219,
      "learning_rate": 4.914294015107496e-06,
      "logits/chosen": -0.8163889050483704,
      "logits/rejected": -0.8286741971969604,
      "logps/chosen": -65.88087463378906,
      "logps/rejected": -69.69749450683594,
      "loss": 14.0756,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.025371646508574486,
      "rewards/margins": -0.01930541917681694,
      "rewards/rejected": -0.006066230591386557,
      "step": 59
    },
    {
      "epoch": 0.03484927687750479,
      "grad_norm": 313.75762939453125,
      "learning_rate": 4.9128413712957585e-06,
      "logits/chosen": -0.8723942041397095,
      "logits/rejected": -0.852526068687439,
      "logps/chosen": -69.1073989868164,
      "logps/rejected": -78.30496978759766,
      "loss": 13.6396,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.007079926319420338,
      "rewards/margins": 0.024318810552358627,
      "rewards/rejected": -0.03139873221516609,
      "step": 60
    },
    {
      "epoch": 0.03543009815879654,
      "grad_norm": 301.9634704589844,
      "learning_rate": 4.911388727484021e-06,
      "logits/chosen": -0.9334823489189148,
      "logits/rejected": -0.8579393625259399,
      "logps/chosen": -68.89993286132812,
      "logps/rejected": -71.83196258544922,
      "loss": 13.8795,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02495257556438446,
      "rewards/margins": 0.0009910565568134189,
      "rewards/rejected": -0.025943631306290627,
      "step": 61
    },
    {
      "epoch": 0.03601091944008829,
      "grad_norm": 312.14678955078125,
      "learning_rate": 4.909936083672284e-06,
      "logits/chosen": -1.0733639001846313,
      "logits/rejected": -0.9336859583854675,
      "logps/chosen": -71.59821319580078,
      "logps/rejected": -83.17411804199219,
      "loss": 13.7566,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03083074651658535,
      "rewards/margins": 0.01191837340593338,
      "rewards/rejected": -0.04274912178516388,
      "step": 62
    },
    {
      "epoch": 0.03659174072138003,
      "grad_norm": 427.48895263671875,
      "learning_rate": 4.908483439860547e-06,
      "logits/chosen": -0.9465745091438293,
      "logits/rejected": -0.9409465789794922,
      "logps/chosen": -77.52535247802734,
      "logps/rejected": -77.35426330566406,
      "loss": 13.8253,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04855041205883026,
      "rewards/margins": 0.005750913638621569,
      "rewards/rejected": -0.05430132895708084,
      "step": 63
    },
    {
      "epoch": 0.03717256200267178,
      "grad_norm": 329.34857177734375,
      "learning_rate": 4.9070307960488095e-06,
      "logits/chosen": -0.8759803771972656,
      "logits/rejected": -0.9234689474105835,
      "logps/chosen": -62.841156005859375,
      "logps/rejected": -66.76082611083984,
      "loss": 13.9698,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.018310727551579475,
      "rewards/margins": -0.008804955519735813,
      "rewards/rejected": -0.009505772963166237,
      "step": 64
    },
    {
      "epoch": 0.03775338328396353,
      "grad_norm": 290.18682861328125,
      "learning_rate": 4.905578152237072e-06,
      "logits/chosen": -0.9876300692558289,
      "logits/rejected": -0.9204443097114563,
      "logps/chosen": -70.43892669677734,
      "logps/rejected": -69.47483825683594,
      "loss": 13.7851,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.017851095646619797,
      "rewards/margins": 0.011618509888648987,
      "rewards/rejected": -0.029469609260559082,
      "step": 65
    },
    {
      "epoch": 0.03833420456525527,
      "grad_norm": 331.4013977050781,
      "learning_rate": 4.904125508425335e-06,
      "logits/chosen": -0.8038953542709351,
      "logits/rejected": -0.7992674112319946,
      "logps/chosen": -67.48692321777344,
      "logps/rejected": -85.4039077758789,
      "loss": 13.6661,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02300168201327324,
      "rewards/margins": 0.022488538175821304,
      "rewards/rejected": -0.045490216463804245,
      "step": 66
    },
    {
      "epoch": 0.03891502584654702,
      "grad_norm": 324.8753356933594,
      "learning_rate": 4.902672864613598e-06,
      "logits/chosen": -0.7837721705436707,
      "logits/rejected": -0.9071874618530273,
      "logps/chosen": -69.61351013183594,
      "logps/rejected": -68.86196899414062,
      "loss": 13.79,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.02017301693558693,
      "rewards/margins": 0.008808745071291924,
      "rewards/rejected": -0.028981763869524002,
      "step": 67
    },
    {
      "epoch": 0.03949584712783876,
      "grad_norm": 317.4655456542969,
      "learning_rate": 4.90122022080186e-06,
      "logits/chosen": -0.8771296739578247,
      "logits/rejected": -0.8569726943969727,
      "logps/chosen": -73.78062438964844,
      "logps/rejected": -68.98119354248047,
      "loss": 13.5934,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.009509051218628883,
      "rewards/margins": 0.02956991270184517,
      "rewards/rejected": -0.0390789620578289,
      "step": 68
    },
    {
      "epoch": 0.04007666840913051,
      "grad_norm": 349.1176452636719,
      "learning_rate": 4.8997675769901224e-06,
      "logits/chosen": -0.8722847700119019,
      "logits/rejected": -0.8638531565666199,
      "logps/chosen": -74.6583251953125,
      "logps/rejected": -73.25667572021484,
      "loss": 13.7595,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.01801307685673237,
      "rewards/margins": 0.012483290396630764,
      "rewards/rejected": -0.030496370047330856,
      "step": 69
    },
    {
      "epoch": 0.04065748969042226,
      "grad_norm": 321.077880859375,
      "learning_rate": 4.898314933178385e-06,
      "logits/chosen": -0.6883363723754883,
      "logits/rejected": -0.6323266625404358,
      "logps/chosen": -73.32432556152344,
      "logps/rejected": -87.47152709960938,
      "loss": 13.6591,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0362272784113884,
      "rewards/margins": 0.024491379037499428,
      "rewards/rejected": -0.060718655586242676,
      "step": 70
    },
    {
      "epoch": 0.041238310971714,
      "grad_norm": 306.60955810546875,
      "learning_rate": 4.896862289366648e-06,
      "logits/chosen": -0.9396616816520691,
      "logits/rejected": -0.8107419013977051,
      "logps/chosen": -75.45014953613281,
      "logps/rejected": -74.94744110107422,
      "loss": 13.6692,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.030824948102235794,
      "rewards/margins": 0.020616179332137108,
      "rewards/rejected": -0.051441121846437454,
      "step": 71
    },
    {
      "epoch": 0.04181913225300575,
      "grad_norm": 309.03094482421875,
      "learning_rate": 4.895409645554911e-06,
      "logits/chosen": -1.0361931324005127,
      "logits/rejected": -0.920698344707489,
      "logps/chosen": -74.54027557373047,
      "logps/rejected": -66.71863555908203,
      "loss": 14.2171,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.06477208435535431,
      "rewards/margins": -0.031630050390958786,
      "rewards/rejected": -0.03314203768968582,
      "step": 72
    },
    {
      "epoch": 0.0423999535342975,
      "grad_norm": 336.6788635253906,
      "learning_rate": 4.893957001743173e-06,
      "logits/chosen": -0.9846957921981812,
      "logits/rejected": -0.9497137069702148,
      "logps/chosen": -73.15393829345703,
      "logps/rejected": -73.5201416015625,
      "loss": 13.9208,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019983595237135887,
      "rewards/margins": -0.003768919501453638,
      "rewards/rejected": -0.016214676201343536,
      "step": 73
    },
    {
      "epoch": 0.04298077481558924,
      "grad_norm": 328.0151062011719,
      "learning_rate": 4.892504357931436e-06,
      "logits/chosen": -0.9070509672164917,
      "logits/rejected": -0.890802264213562,
      "logps/chosen": -74.13055419921875,
      "logps/rejected": -74.56624603271484,
      "loss": 14.2509,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03255216404795647,
      "rewards/margins": -0.03526480868458748,
      "rewards/rejected": 0.0027126409113407135,
      "step": 74
    },
    {
      "epoch": 0.04356159609688099,
      "grad_norm": 312.4993591308594,
      "learning_rate": 4.891051714119698e-06,
      "logits/chosen": -0.8348041772842407,
      "logits/rejected": -0.8501715660095215,
      "logps/chosen": -76.722900390625,
      "logps/rejected": -71.301025390625,
      "loss": 14.0253,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024392826482653618,
      "rewards/margins": -0.01172790676355362,
      "rewards/rejected": -0.012664918787777424,
      "step": 75
    },
    {
      "epoch": 0.04414241737817274,
      "grad_norm": 308.2872314453125,
      "learning_rate": 4.889599070307961e-06,
      "logits/chosen": -0.5838706493377686,
      "logits/rejected": -0.637297511100769,
      "logps/chosen": -72.34232330322266,
      "logps/rejected": -66.2028579711914,
      "loss": 14.1823,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -0.019541073590517044,
      "rewards/margins": -0.02966020628809929,
      "rewards/rejected": 0.010119132697582245,
      "step": 76
    },
    {
      "epoch": 0.04472323865946448,
      "grad_norm": 305.4457702636719,
      "learning_rate": 4.8881464264962236e-06,
      "logits/chosen": -0.973824143409729,
      "logits/rejected": -0.9775202870368958,
      "logps/chosen": -72.74274444580078,
      "logps/rejected": -73.25370788574219,
      "loss": 14.0823,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.02193290740251541,
      "rewards/margins": -0.019592974334955215,
      "rewards/rejected": -0.0023399335332214832,
      "step": 77
    },
    {
      "epoch": 0.04530405994075623,
      "grad_norm": 483.3565368652344,
      "learning_rate": 4.886693782684486e-06,
      "logits/chosen": -0.8660491704940796,
      "logits/rejected": -0.8472667932510376,
      "logps/chosen": -79.67647552490234,
      "logps/rejected": -76.70512390136719,
      "loss": 14.0415,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.035908013582229614,
      "rewards/margins": -0.01337387878447771,
      "rewards/rejected": -0.02253413386642933,
      "step": 78
    },
    {
      "epoch": 0.04588488122204797,
      "grad_norm": 315.49755859375,
      "learning_rate": 4.885241138872749e-06,
      "logits/chosen": -0.6490969061851501,
      "logits/rejected": -0.7820181250572205,
      "logps/chosen": -74.25728607177734,
      "logps/rejected": -73.80535125732422,
      "loss": 13.4779,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.014750528149306774,
      "rewards/margins": 0.043530743569135666,
      "rewards/rejected": -0.058281272649765015,
      "step": 79
    },
    {
      "epoch": 0.04646570250333972,
      "grad_norm": 315.58837890625,
      "learning_rate": 4.883788495061012e-06,
      "logits/chosen": -0.8913451433181763,
      "logits/rejected": -0.8876463770866394,
      "logps/chosen": -73.64271545410156,
      "logps/rejected": -69.37626647949219,
      "loss": 13.957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01223234087228775,
      "rewards/margins": -0.005636455025523901,
      "rewards/rejected": -0.006595888640731573,
      "step": 80
    },
    {
      "epoch": 0.04704652378463147,
      "grad_norm": 313.813232421875,
      "learning_rate": 4.882335851249274e-06,
      "logits/chosen": -0.8722120523452759,
      "logits/rejected": -0.7989141345024109,
      "logps/chosen": -70.44036865234375,
      "logps/rejected": -70.73589324951172,
      "loss": 14.1477,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.026375526562333107,
      "rewards/margins": -0.026495525613427162,
      "rewards/rejected": 0.00012000077549600974,
      "step": 81
    },
    {
      "epoch": 0.04762734506592321,
      "grad_norm": 314.9659729003906,
      "learning_rate": 4.8808832074375365e-06,
      "logits/chosen": -0.7730615735054016,
      "logits/rejected": -0.8082054257392883,
      "logps/chosen": -69.96324157714844,
      "logps/rejected": -77.90928649902344,
      "loss": 13.778,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.014157066121697426,
      "rewards/margins": 0.010449771769344807,
      "rewards/rejected": -0.024606838822364807,
      "step": 82
    },
    {
      "epoch": 0.04820816634721496,
      "grad_norm": 303.14599609375,
      "learning_rate": 4.879430563625799e-06,
      "logits/chosen": -0.9874809980392456,
      "logits/rejected": -1.0569359064102173,
      "logps/chosen": -74.83180236816406,
      "logps/rejected": -76.95542907714844,
      "loss": 13.6937,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0425366535782814,
      "rewards/margins": 0.021705975756049156,
      "rewards/rejected": -0.06424263119697571,
      "step": 83
    },
    {
      "epoch": 0.04878898762850671,
      "grad_norm": 317.16961669921875,
      "learning_rate": 4.877977919814062e-06,
      "logits/chosen": -0.9083768725395203,
      "logits/rejected": -0.9116488695144653,
      "logps/chosen": -80.80293273925781,
      "logps/rejected": -72.5535888671875,
      "loss": 14.2559,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.06587149202823639,
      "rewards/margins": -0.036308903247117996,
      "rewards/rejected": -0.029562586918473244,
      "step": 84
    },
    {
      "epoch": 0.04936980890979845,
      "grad_norm": 301.5,
      "learning_rate": 4.876525276002325e-06,
      "logits/chosen": -0.9549547433853149,
      "logits/rejected": -0.9115845561027527,
      "logps/chosen": -75.24485778808594,
      "logps/rejected": -68.90257263183594,
      "loss": 14.065,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.015530148521065712,
      "rewards/margins": -0.0176745243370533,
      "rewards/rejected": 0.0021443753503262997,
      "step": 85
    },
    {
      "epoch": 0.0499506301910902,
      "grad_norm": 307.9525451660156,
      "learning_rate": 4.8750726321905875e-06,
      "logits/chosen": -0.9091174006462097,
      "logits/rejected": -0.9419649839401245,
      "logps/chosen": -76.40791320800781,
      "logps/rejected": -77.6815185546875,
      "loss": 13.7705,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.015883130952715874,
      "rewards/margins": 0.01122428011149168,
      "rewards/rejected": -0.02710741199553013,
      "step": 86
    },
    {
      "epoch": 0.05053145147238195,
      "grad_norm": 321.51507568359375,
      "learning_rate": 4.87361998837885e-06,
      "logits/chosen": -0.9909089803695679,
      "logits/rejected": -0.9910022616386414,
      "logps/chosen": -77.63959503173828,
      "logps/rejected": -72.30487823486328,
      "loss": 13.8477,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.006782358977943659,
      "rewards/margins": 0.003400030778720975,
      "rewards/rejected": -0.010182389989495277,
      "step": 87
    },
    {
      "epoch": 0.05111227275367369,
      "grad_norm": 321.3612060546875,
      "learning_rate": 4.872167344567112e-06,
      "logits/chosen": -0.7301020622253418,
      "logits/rejected": -0.5697265267372131,
      "logps/chosen": -70.13436126708984,
      "logps/rejected": -77.76301574707031,
      "loss": 13.7418,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.015415447764098644,
      "rewards/margins": 0.013852661475539207,
      "rewards/rejected": 0.0015627862885594368,
      "step": 88
    },
    {
      "epoch": 0.05169309403496544,
      "grad_norm": 322.7413330078125,
      "learning_rate": 4.870714700755375e-06,
      "logits/chosen": -0.8953542709350586,
      "logits/rejected": -0.8882652521133423,
      "logps/chosen": -69.642333984375,
      "logps/rejected": -66.21636962890625,
      "loss": 13.8948,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015242251567542553,
      "rewards/margins": -0.00036549606011249125,
      "rewards/rejected": -0.014876757748425007,
      "step": 89
    },
    {
      "epoch": 0.052273915316257184,
      "grad_norm": 316.21905517578125,
      "learning_rate": 4.869262056943638e-06,
      "logits/chosen": -0.826396107673645,
      "logits/rejected": -0.8409526944160461,
      "logps/chosen": -84.0012435913086,
      "logps/rejected": -73.15267944335938,
      "loss": 14.2303,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.03959153965115547,
      "rewards/margins": -0.03513690084218979,
      "rewards/rejected": -0.004454641602933407,
      "step": 90
    },
    {
      "epoch": 0.05285473659754893,
      "grad_norm": 314.6103515625,
      "learning_rate": 4.8678094131319e-06,
      "logits/chosen": -0.9043526649475098,
      "logits/rejected": -0.9540117979049683,
      "logps/chosen": -85.2854995727539,
      "logps/rejected": -74.1194076538086,
      "loss": 14.256,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.05172392725944519,
      "rewards/margins": -0.03560470789670944,
      "rewards/rejected": -0.016119223088026047,
      "step": 91
    },
    {
      "epoch": 0.05343555787884068,
      "grad_norm": 298.3580322265625,
      "learning_rate": 4.866356769320163e-06,
      "logits/chosen": -0.83808434009552,
      "logits/rejected": -0.858515739440918,
      "logps/chosen": -67.17420959472656,
      "logps/rejected": -84.23038482666016,
      "loss": 13.4033,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0225966926664114,
      "rewards/margins": 0.05055863410234451,
      "rewards/rejected": -0.027961939573287964,
      "step": 92
    },
    {
      "epoch": 0.054016379160132424,
      "grad_norm": 293.33282470703125,
      "learning_rate": 4.864904125508426e-06,
      "logits/chosen": -0.9872828722000122,
      "logits/rejected": -1.1029959917068481,
      "logps/chosen": -72.00138854980469,
      "logps/rejected": -69.6135025024414,
      "loss": 13.7105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0009010225767269731,
      "rewards/margins": 0.017413010820746422,
      "rewards/rejected": -0.01831403188407421,
      "step": 93
    },
    {
      "epoch": 0.054597200441424174,
      "grad_norm": 356.63946533203125,
      "learning_rate": 4.863451481696689e-06,
      "logits/chosen": -0.8439090847969055,
      "logits/rejected": -0.8643622398376465,
      "logps/chosen": -70.82389831542969,
      "logps/rejected": -75.1737289428711,
      "loss": 13.8326,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0158695038408041,
      "rewards/margins": 0.0050217146053910255,
      "rewards/rejected": -0.0208912193775177,
      "step": 94
    },
    {
      "epoch": 0.05517802172271592,
      "grad_norm": 304.1072998046875,
      "learning_rate": 4.8619988378849505e-06,
      "logits/chosen": -1.0255842208862305,
      "logits/rejected": -1.041534662246704,
      "logps/chosen": -68.80192565917969,
      "logps/rejected": -67.79802703857422,
      "loss": 13.6247,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01212473027408123,
      "rewards/margins": 0.02532658539712429,
      "rewards/rejected": -0.01320185698568821,
      "step": 95
    },
    {
      "epoch": 0.055758843004007665,
      "grad_norm": 314.3151550292969,
      "learning_rate": 4.860546194073213e-06,
      "logits/chosen": -0.8595132827758789,
      "logits/rejected": -0.8583124279975891,
      "logps/chosen": -72.1944808959961,
      "logps/rejected": -72.77186584472656,
      "loss": 13.9049,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.031448423862457275,
      "rewards/margins": -0.0011334316805005074,
      "rewards/rejected": -0.030314987525343895,
      "step": 96
    },
    {
      "epoch": 0.056339664285299414,
      "grad_norm": 320.2240905761719,
      "learning_rate": 4.859093550261476e-06,
      "logits/chosen": -0.749364972114563,
      "logits/rejected": -0.8103491067886353,
      "logps/chosen": -79.1120376586914,
      "logps/rejected": -75.91130065917969,
      "loss": 13.9032,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03181576728820801,
      "rewards/margins": -0.0017790347337722778,
      "rewards/rejected": -0.03003673627972603,
      "step": 97
    },
    {
      "epoch": 0.05692048556659116,
      "grad_norm": 314.99896240234375,
      "learning_rate": 4.857640906449739e-06,
      "logits/chosen": -0.8008295297622681,
      "logits/rejected": -0.847716212272644,
      "logps/chosen": -79.81649017333984,
      "logps/rejected": -68.53919982910156,
      "loss": 14.0979,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.021854501217603683,
      "rewards/margins": -0.02203025482594967,
      "rewards/rejected": 0.00017575845413375646,
      "step": 98
    },
    {
      "epoch": 0.057501306847882905,
      "grad_norm": 315.1274108886719,
      "learning_rate": 4.8561882626380015e-06,
      "logits/chosen": -0.8115674257278442,
      "logits/rejected": -0.8901892900466919,
      "logps/chosen": -72.59163665771484,
      "logps/rejected": -78.07933044433594,
      "loss": 13.7458,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.000810365192592144,
      "rewards/margins": 0.014865818433463573,
      "rewards/rejected": -0.01405545137822628,
      "step": 99
    },
    {
      "epoch": 0.058082128129174654,
      "grad_norm": 318.1490173339844,
      "learning_rate": 4.854735618826264e-06,
      "logits/chosen": -0.9334144592285156,
      "logits/rejected": -0.9743694067001343,
      "logps/chosen": -74.79545593261719,
      "logps/rejected": -80.59342193603516,
      "loss": 13.5047,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.00012166835222160444,
      "rewards/margins": 0.038393907248973846,
      "rewards/rejected": -0.03827223926782608,
      "step": 100
    },
    {
      "epoch": 0.0586629494104664,
      "grad_norm": 312.1184997558594,
      "learning_rate": 4.853282975014527e-06,
      "logits/chosen": -0.8218280076980591,
      "logits/rejected": -0.848983645439148,
      "logps/chosen": -77.64916229248047,
      "logps/rejected": -77.53215026855469,
      "loss": 13.6869,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0032627955079078674,
      "rewards/margins": 0.022136736661195755,
      "rewards/rejected": -0.025399532169103622,
      "step": 101
    },
    {
      "epoch": 0.059243770691758145,
      "grad_norm": 314.2687072753906,
      "learning_rate": 4.851830331202789e-06,
      "logits/chosen": -0.8590647578239441,
      "logits/rejected": -0.8945513963699341,
      "logps/chosen": -78.69733428955078,
      "logps/rejected": -68.73847198486328,
      "loss": 13.7324,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010157021693885326,
      "rewards/margins": 0.016794661059975624,
      "rewards/rejected": -0.026951681822538376,
      "step": 102
    },
    {
      "epoch": 0.059824591973049894,
      "grad_norm": 322.7899475097656,
      "learning_rate": 4.850377687391052e-06,
      "logits/chosen": -0.9431624412536621,
      "logits/rejected": -0.8844934701919556,
      "logps/chosen": -71.08782958984375,
      "logps/rejected": -76.45503234863281,
      "loss": 13.7423,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0048022083938121796,
      "rewards/margins": 0.014014339074492455,
      "rewards/rejected": -0.018816547468304634,
      "step": 103
    },
    {
      "epoch": 0.060405413254341636,
      "grad_norm": 328.67974853515625,
      "learning_rate": 4.848925043579314e-06,
      "logits/chosen": -0.7548056840896606,
      "logits/rejected": -0.7886452078819275,
      "logps/chosen": -74.668212890625,
      "logps/rejected": -74.39926147460938,
      "loss": 13.3898,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.050213318318128586,
      "rewards/margins": 0.04900939390063286,
      "rewards/rejected": 0.0012039269786328077,
      "step": 104
    },
    {
      "epoch": 0.060986234535633385,
      "grad_norm": 306.2090148925781,
      "learning_rate": 4.847472399767578e-06,
      "logits/chosen": -0.7457195520401001,
      "logits/rejected": -0.7296000123023987,
      "logps/chosen": -66.86776733398438,
      "logps/rejected": -68.6021499633789,
      "loss": 14.2794,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.01881163753569126,
      "rewards/margins": -0.039370544254779816,
      "rewards/rejected": 0.020558910444378853,
      "step": 105
    },
    {
      "epoch": 0.061567055816925134,
      "grad_norm": 429.5802917480469,
      "learning_rate": 4.84601975595584e-06,
      "logits/chosen": -0.7268589735031128,
      "logits/rejected": -0.824454665184021,
      "logps/chosen": -73.6258773803711,
      "logps/rejected": -73.26065826416016,
      "loss": 13.8393,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.019226079806685448,
      "rewards/margins": 0.005577159114181995,
      "rewards/rejected": -0.02480323798954487,
      "step": 106
    },
    {
      "epoch": 0.062147877098216876,
      "grad_norm": 310.4759521484375,
      "learning_rate": 4.844567112144103e-06,
      "logits/chosen": -0.7717695236206055,
      "logits/rejected": -0.6804165244102478,
      "logps/chosen": -73.10393524169922,
      "logps/rejected": -71.20040893554688,
      "loss": 13.5079,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.018317507579922676,
      "rewards/margins": 0.03789940103888512,
      "rewards/rejected": -0.05621690675616264,
      "step": 107
    },
    {
      "epoch": 0.06272869837950862,
      "grad_norm": 313.2925720214844,
      "learning_rate": 4.843114468332365e-06,
      "logits/chosen": -0.7994370460510254,
      "logits/rejected": -0.9286397099494934,
      "logps/chosen": -73.3687515258789,
      "logps/rejected": -73.391357421875,
      "loss": 13.7167,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0021995368879288435,
      "rewards/margins": 0.017970655113458633,
      "rewards/rejected": -0.020170193165540695,
      "step": 108
    },
    {
      "epoch": 0.06330951966080037,
      "grad_norm": 298.9688415527344,
      "learning_rate": 4.841661824520628e-06,
      "logits/chosen": -0.783902108669281,
      "logits/rejected": -0.8706483840942383,
      "logps/chosen": -72.7656021118164,
      "logps/rejected": -67.734619140625,
      "loss": 14.0378,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.010019464418292046,
      "rewards/margins": -0.016036922112107277,
      "rewards/rejected": 0.006017456762492657,
      "step": 109
    },
    {
      "epoch": 0.06389034094209212,
      "grad_norm": 292.80926513671875,
      "learning_rate": 4.840209180708891e-06,
      "logits/chosen": -0.867302417755127,
      "logits/rejected": -0.934320330619812,
      "logps/chosen": -69.18145751953125,
      "logps/rejected": -74.14984893798828,
      "loss": 13.9161,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04971027001738548,
      "rewards/margins": -0.003587187733501196,
      "rewards/rejected": -0.04612307995557785,
      "step": 110
    },
    {
      "epoch": 0.06447116222338387,
      "grad_norm": 316.4573059082031,
      "learning_rate": 4.838756536897154e-06,
      "logits/chosen": -0.7158193588256836,
      "logits/rejected": -0.703850269317627,
      "logps/chosen": -72.75650024414062,
      "logps/rejected": -74.89552307128906,
      "loss": 13.7563,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02667677029967308,
      "rewards/margins": 0.013364049606025219,
      "rewards/rejected": 0.013312721624970436,
      "step": 111
    },
    {
      "epoch": 0.06505198350467561,
      "grad_norm": 310.0760498046875,
      "learning_rate": 4.837303893085416e-06,
      "logits/chosen": -0.7871710658073425,
      "logits/rejected": -0.7631909847259521,
      "logps/chosen": -72.4153823852539,
      "logps/rejected": -69.98908233642578,
      "loss": 14.0832,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.027912933379411697,
      "rewards/margins": -0.020552167668938637,
      "rewards/rejected": -0.007360764779150486,
      "step": 112
    },
    {
      "epoch": 0.06563280478596736,
      "grad_norm": 298.72796630859375,
      "learning_rate": 4.835851249273678e-06,
      "logits/chosen": -0.8587129712104797,
      "logits/rejected": -0.828883171081543,
      "logps/chosen": -76.20726013183594,
      "logps/rejected": -70.39952087402344,
      "loss": 13.7695,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01639840006828308,
      "rewards/margins": 0.01160570327192545,
      "rewards/rejected": 0.004792699124664068,
      "step": 113
    },
    {
      "epoch": 0.0662136260672591,
      "grad_norm": 306.0551452636719,
      "learning_rate": 4.834398605461941e-06,
      "logits/chosen": -0.7858158349990845,
      "logits/rejected": -0.8138400912284851,
      "logps/chosen": -71.29947662353516,
      "logps/rejected": -74.6938247680664,
      "loss": 13.6944,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.009441891685128212,
      "rewards/margins": 0.020461464300751686,
      "rewards/rejected": -0.011019574478268623,
      "step": 114
    },
    {
      "epoch": 0.06679444734855085,
      "grad_norm": 299.12298583984375,
      "learning_rate": 4.832945961650204e-06,
      "logits/chosen": -0.8294021487236023,
      "logits/rejected": -0.9068562388420105,
      "logps/chosen": -72.25274658203125,
      "logps/rejected": -74.01802062988281,
      "loss": 13.8028,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.007599563803523779,
      "rewards/margins": 0.009034421294927597,
      "rewards/rejected": -0.01663398928940296,
      "step": 115
    },
    {
      "epoch": 0.0673752686298426,
      "grad_norm": 316.1508483886719,
      "learning_rate": 4.8314933178384665e-06,
      "logits/chosen": -0.7372664213180542,
      "logits/rejected": -0.6621009707450867,
      "logps/chosen": -71.98650360107422,
      "logps/rejected": -73.5616226196289,
      "loss": 14.0602,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.040212083607912064,
      "rewards/margins": -0.01578274928033352,
      "rewards/rejected": -0.024429330602288246,
      "step": 116
    },
    {
      "epoch": 0.06795608991113435,
      "grad_norm": 311.5640869140625,
      "learning_rate": 4.830040674026729e-06,
      "logits/chosen": -0.8206332325935364,
      "logits/rejected": -0.8395715951919556,
      "logps/chosen": -71.33819580078125,
      "logps/rejected": -67.67076110839844,
      "loss": 14.0576,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04526720941066742,
      "rewards/margins": -0.017988480627536774,
      "rewards/rejected": -0.027278726920485497,
      "step": 117
    },
    {
      "epoch": 0.0685369111924261,
      "grad_norm": 319.0810852050781,
      "learning_rate": 4.828588030214992e-06,
      "logits/chosen": -0.75376957654953,
      "logits/rejected": -0.7071677446365356,
      "logps/chosen": -79.94001770019531,
      "logps/rejected": -73.17815399169922,
      "loss": 14.162,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.015543567016720772,
      "rewards/margins": -0.02663475275039673,
      "rewards/rejected": 0.011091184802353382,
      "step": 118
    },
    {
      "epoch": 0.06911773247371784,
      "grad_norm": 296.51690673828125,
      "learning_rate": 4.827135386403255e-06,
      "logits/chosen": -0.8419657945632935,
      "logits/rejected": -0.7793577313423157,
      "logps/chosen": -72.96027374267578,
      "logps/rejected": -71.80851745605469,
      "loss": 13.2528,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.016827832907438278,
      "rewards/margins": 0.06467505544424057,
      "rewards/rejected": -0.047847211360931396,
      "step": 119
    },
    {
      "epoch": 0.06969855375500958,
      "grad_norm": 305.2845458984375,
      "learning_rate": 4.825682742591517e-06,
      "logits/chosen": -0.7169400453567505,
      "logits/rejected": -0.7394998073577881,
      "logps/chosen": -69.85813903808594,
      "logps/rejected": -76.11322784423828,
      "loss": 13.4805,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0008814238244667649,
      "rewards/margins": 0.041327688843011856,
      "rewards/rejected": -0.04220911115407944,
      "step": 120
    },
    {
      "epoch": 0.07027937503630133,
      "grad_norm": 329.4168395996094,
      "learning_rate": 4.824230098779779e-06,
      "logits/chosen": -0.8370596170425415,
      "logits/rejected": -0.8346614837646484,
      "logps/chosen": -76.3817138671875,
      "logps/rejected": -69.7574691772461,
      "loss": 14.3016,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.017085228115320206,
      "rewards/margins": -0.039855439215898514,
      "rewards/rejected": 0.02277020923793316,
      "step": 121
    },
    {
      "epoch": 0.07086019631759308,
      "grad_norm": 328.85821533203125,
      "learning_rate": 4.822777454968042e-06,
      "logits/chosen": -0.8888875246047974,
      "logits/rejected": -0.8640028834342957,
      "logps/chosen": -81.20426940917969,
      "logps/rejected": -73.75362396240234,
      "loss": 14.2056,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05541349574923515,
      "rewards/margins": -0.030969763174653053,
      "rewards/rejected": -0.02444373071193695,
      "step": 122
    },
    {
      "epoch": 0.07144101759888483,
      "grad_norm": 329.655517578125,
      "learning_rate": 4.821324811156305e-06,
      "logits/chosen": -0.8150280714035034,
      "logits/rejected": -0.7874841094017029,
      "logps/chosen": -74.91133880615234,
      "logps/rejected": -76.65450286865234,
      "loss": 13.9383,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04368335008621216,
      "rewards/margins": -0.005534342024475336,
      "rewards/rejected": -0.03814900666475296,
      "step": 123
    },
    {
      "epoch": 0.07202183888017658,
      "grad_norm": 287.1902770996094,
      "learning_rate": 4.819872167344568e-06,
      "logits/chosen": -0.9967269897460938,
      "logits/rejected": -1.0370653867721558,
      "logps/chosen": -73.49361419677734,
      "logps/rejected": -76.58966827392578,
      "loss": 13.7119,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.034154172986745834,
      "rewards/margins": 0.019786100834608078,
      "rewards/rejected": -0.05394027382135391,
      "step": 124
    },
    {
      "epoch": 0.07260266016146831,
      "grad_norm": 321.77142333984375,
      "learning_rate": 4.81841952353283e-06,
      "logits/chosen": -0.8000070452690125,
      "logits/rejected": -0.8691812753677368,
      "logps/chosen": -77.01860809326172,
      "logps/rejected": -73.41801452636719,
      "loss": 13.4661,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.00704178074374795,
      "rewards/margins": 0.04257983714342117,
      "rewards/rejected": -0.04962162673473358,
      "step": 125
    },
    {
      "epoch": 0.07318348144276006,
      "grad_norm": 329.365478515625,
      "learning_rate": 4.816966879721093e-06,
      "logits/chosen": -0.6257106065750122,
      "logits/rejected": -0.7529661059379578,
      "logps/chosen": -70.44479370117188,
      "logps/rejected": -71.65373229980469,
      "loss": 13.9067,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.030417144298553467,
      "rewards/margins": -0.0017956402152776718,
      "rewards/rejected": -0.028621505945920944,
      "step": 126
    },
    {
      "epoch": 0.07376430272405181,
      "grad_norm": 305.7592468261719,
      "learning_rate": 4.815514235909355e-06,
      "logits/chosen": -0.8345580101013184,
      "logits/rejected": -0.809880256652832,
      "logps/chosen": -75.27732849121094,
      "logps/rejected": -76.76701354980469,
      "loss": 13.7657,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02379775047302246,
      "rewards/margins": 0.011454248800873756,
      "rewards/rejected": -0.03525200113654137,
      "step": 127
    },
    {
      "epoch": 0.07434512400534356,
      "grad_norm": 326.51171875,
      "learning_rate": 4.814061592097618e-06,
      "logits/chosen": -0.8751303553581238,
      "logits/rejected": -0.7346702814102173,
      "logps/chosen": -71.24671936035156,
      "logps/rejected": -76.22618103027344,
      "loss": 13.7425,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.002272081095725298,
      "rewards/margins": 0.013138952665030956,
      "rewards/rejected": -0.010866871103644371,
      "step": 128
    },
    {
      "epoch": 0.0749259452866353,
      "grad_norm": 331.1041259765625,
      "learning_rate": 4.8126089482858805e-06,
      "logits/chosen": -0.840943455696106,
      "logits/rejected": -0.8373934626579285,
      "logps/chosen": -74.03942108154297,
      "logps/rejected": -69.12279510498047,
      "loss": 14.2463,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0546981580555439,
      "rewards/margins": -0.03357243537902832,
      "rewards/rejected": -0.02112571895122528,
      "step": 129
    },
    {
      "epoch": 0.07550676656792706,
      "grad_norm": 312.5914611816406,
      "learning_rate": 4.811156304474143e-06,
      "logits/chosen": -0.8838016390800476,
      "logits/rejected": -0.9450550079345703,
      "logps/chosen": -79.87310028076172,
      "logps/rejected": -80.9448013305664,
      "loss": 13.7435,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03348521143198013,
      "rewards/margins": 0.014457473531365395,
      "rewards/rejected": -0.04794268682599068,
      "step": 130
    },
    {
      "epoch": 0.07608758784921879,
      "grad_norm": 291.64898681640625,
      "learning_rate": 4.809703660662406e-06,
      "logits/chosen": -0.9270390272140503,
      "logits/rejected": -1.0079596042633057,
      "logps/chosen": -68.3660659790039,
      "logps/rejected": -75.16285705566406,
      "loss": 13.761,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01317480206489563,
      "rewards/margins": 0.013511622324585915,
      "rewards/rejected": -0.026686420664191246,
      "step": 131
    },
    {
      "epoch": 0.07666840913051054,
      "grad_norm": 325.22186279296875,
      "learning_rate": 4.808251016850669e-06,
      "logits/chosen": -0.793510913848877,
      "logits/rejected": -0.8536527752876282,
      "logps/chosen": -79.37129211425781,
      "logps/rejected": -81.64805603027344,
      "loss": 13.9288,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04622223600745201,
      "rewards/margins": -0.0038099661469459534,
      "rewards/rejected": -0.042412273585796356,
      "step": 132
    },
    {
      "epoch": 0.07724923041180229,
      "grad_norm": 332.0003662109375,
      "learning_rate": 4.8067983730389315e-06,
      "logits/chosen": -0.863021731376648,
      "logits/rejected": -0.7807949185371399,
      "logps/chosen": -70.98688507080078,
      "logps/rejected": -74.82456970214844,
      "loss": 13.6363,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03255782276391983,
      "rewards/margins": 0.027058040723204613,
      "rewards/rejected": -0.059615861624479294,
      "step": 133
    },
    {
      "epoch": 0.07783005169309404,
      "grad_norm": 419.6676025390625,
      "learning_rate": 4.8053457292271934e-06,
      "logits/chosen": -0.6968336701393127,
      "logits/rejected": -0.8155566453933716,
      "logps/chosen": -76.53175354003906,
      "logps/rejected": -80.69217681884766,
      "loss": 13.5714,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.003611106425523758,
      "rewards/margins": 0.0325997918844223,
      "rewards/rejected": -0.03621090203523636,
      "step": 134
    },
    {
      "epoch": 0.07841087297438579,
      "grad_norm": 318.14093017578125,
      "learning_rate": 4.803893085415456e-06,
      "logits/chosen": -0.8911747932434082,
      "logits/rejected": -0.8688879013061523,
      "logps/chosen": -78.88302612304688,
      "logps/rejected": -72.34574890136719,
      "loss": 13.7806,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.009762173518538475,
      "rewards/margins": 0.012868879362940788,
      "rewards/rejected": -0.022631052881479263,
      "step": 135
    },
    {
      "epoch": 0.07899169425567752,
      "grad_norm": 301.05181884765625,
      "learning_rate": 4.802440441603719e-06,
      "logits/chosen": -0.9665401577949524,
      "logits/rejected": -1.0596462488174438,
      "logps/chosen": -69.1599349975586,
      "logps/rejected": -72.39261627197266,
      "loss": 13.9663,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006733216345310211,
      "rewards/margins": -0.007116011343896389,
      "rewards/rejected": 0.00038279517320916057,
      "step": 136
    },
    {
      "epoch": 0.07957251553696927,
      "grad_norm": 318.62030029296875,
      "learning_rate": 4.800987797791982e-06,
      "logits/chosen": -0.7195813059806824,
      "logits/rejected": -0.7145063281059265,
      "logps/chosen": -79.5794448852539,
      "logps/rejected": -84.13545227050781,
      "loss": 13.4993,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.002331190975382924,
      "rewards/margins": 0.03869814798235893,
      "rewards/rejected": -0.04102934151887894,
      "step": 137
    },
    {
      "epoch": 0.08015333681826102,
      "grad_norm": 425.3757019042969,
      "learning_rate": 4.7995351539802444e-06,
      "logits/chosen": -0.8645333051681519,
      "logits/rejected": -0.7608574032783508,
      "logps/chosen": -69.97611999511719,
      "logps/rejected": -85.6728515625,
      "loss": 13.1834,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011998976580798626,
      "rewards/margins": 0.07343053817749023,
      "rewards/rejected": -0.06143154948949814,
      "step": 138
    },
    {
      "epoch": 0.08073415809955277,
      "grad_norm": 312.2695007324219,
      "learning_rate": 4.798082510168507e-06,
      "logits/chosen": -0.7758678197860718,
      "logits/rejected": -0.7573307752609253,
      "logps/chosen": -76.74276733398438,
      "logps/rejected": -70.28133392333984,
      "loss": 13.8316,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0051659406162798405,
      "rewards/margins": 0.006639555096626282,
      "rewards/rejected": -0.0014736183220520616,
      "step": 139
    },
    {
      "epoch": 0.08131497938084452,
      "grad_norm": 315.1054992675781,
      "learning_rate": 4.79662986635677e-06,
      "logits/chosen": -0.718429684638977,
      "logits/rejected": -0.8549942970275879,
      "logps/chosen": -70.87870788574219,
      "logps/rejected": -68.09158325195312,
      "loss": 13.8946,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.027258872985839844,
      "rewards/margins": -0.0010386653011664748,
      "rewards/rejected": -0.026220208033919334,
      "step": 140
    },
    {
      "epoch": 0.08189580066213627,
      "grad_norm": 315.59136962890625,
      "learning_rate": 4.795177222545032e-06,
      "logits/chosen": -0.8170045614242554,
      "logits/rejected": -0.7605774402618408,
      "logps/chosen": -69.022705078125,
      "logps/rejected": -81.53439331054688,
      "loss": 13.625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.006630831863731146,
      "rewards/margins": 0.02589798904955387,
      "rewards/rejected": -0.019267160445451736,
      "step": 141
    },
    {
      "epoch": 0.082476621943428,
      "grad_norm": 303.656982421875,
      "learning_rate": 4.793724578733295e-06,
      "logits/chosen": -0.9443836212158203,
      "logits/rejected": -0.944778561592102,
      "logps/chosen": -77.53956604003906,
      "logps/rejected": -78.73460388183594,
      "loss": 13.8955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02790834940969944,
      "rewards/margins": -0.0015594146680086851,
      "rewards/rejected": -0.026348933577537537,
      "step": 142
    },
    {
      "epoch": 0.08305744322471975,
      "grad_norm": 316.501220703125,
      "learning_rate": 4.792271934921557e-06,
      "logits/chosen": -0.7522753477096558,
      "logits/rejected": -0.8681309819221497,
      "logps/chosen": -77.05738830566406,
      "logps/rejected": -70.8436508178711,
      "loss": 14.4085,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.031116142868995667,
      "rewards/margins": -0.05178617313504219,
      "rewards/rejected": 0.020670032128691673,
      "step": 143
    },
    {
      "epoch": 0.0836382645060115,
      "grad_norm": 673.7465209960938,
      "learning_rate": 4.79081929110982e-06,
      "logits/chosen": -0.8983888626098633,
      "logits/rejected": -0.8548039197921753,
      "logps/chosen": -78.78826141357422,
      "logps/rejected": -71.03697204589844,
      "loss": 13.8593,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.004244650714099407,
      "rewards/margins": 0.004877415020018816,
      "rewards/rejected": -0.00912206619977951,
      "step": 144
    },
    {
      "epoch": 0.08421908578730325,
      "grad_norm": 308.432861328125,
      "learning_rate": 4.789366647298083e-06,
      "logits/chosen": -0.8701874613761902,
      "logits/rejected": -0.8173721432685852,
      "logps/chosen": -73.85729217529297,
      "logps/rejected": -74.20492553710938,
      "loss": 13.6175,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.01854265108704567,
      "rewards/margins": 0.02803659439086914,
      "rewards/rejected": -0.04657924920320511,
      "step": 145
    },
    {
      "epoch": 0.084799907068595,
      "grad_norm": 295.6097717285156,
      "learning_rate": 4.787914003486346e-06,
      "logits/chosen": -0.8499331474304199,
      "logits/rejected": -0.8638921976089478,
      "logps/chosen": -66.98426818847656,
      "logps/rejected": -65.90644836425781,
      "loss": 13.8752,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0066559212282299995,
      "rewards/margins": 0.0012805939186364412,
      "rewards/rejected": -0.007936513982713223,
      "step": 146
    },
    {
      "epoch": 0.08538072834988673,
      "grad_norm": 331.2653503417969,
      "learning_rate": 4.786461359674608e-06,
      "logits/chosen": -0.9896368980407715,
      "logits/rejected": -1.1149613857269287,
      "logps/chosen": -77.21143341064453,
      "logps/rejected": -85.37650299072266,
      "loss": 13.5235,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.003070878330618143,
      "rewards/margins": 0.03570512682199478,
      "rewards/rejected": -0.038776006549596786,
      "step": 147
    },
    {
      "epoch": 0.08596154963117848,
      "grad_norm": 356.3915710449219,
      "learning_rate": 4.785008715862871e-06,
      "logits/chosen": -0.8506426811218262,
      "logits/rejected": -0.7640115022659302,
      "logps/chosen": -67.30644989013672,
      "logps/rejected": -71.46052551269531,
      "loss": 13.6259,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.010771533474326134,
      "rewards/margins": 0.026811867952346802,
      "rewards/rejected": -0.037583399564027786,
      "step": 148
    },
    {
      "epoch": 0.08654237091247023,
      "grad_norm": 303.959716796875,
      "learning_rate": 4.783556072051134e-06,
      "logits/chosen": -0.9652125239372253,
      "logits/rejected": -1.0568349361419678,
      "logps/chosen": -78.24043273925781,
      "logps/rejected": -82.4593734741211,
      "loss": 13.4476,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0065461816266179085,
      "rewards/margins": 0.049500562250614166,
      "rewards/rejected": -0.04295438155531883,
      "step": 149
    },
    {
      "epoch": 0.08712319219376198,
      "grad_norm": 301.6387939453125,
      "learning_rate": 4.7821034282393966e-06,
      "logits/chosen": -0.9109539985656738,
      "logits/rejected": -0.8469412922859192,
      "logps/chosen": -71.34504699707031,
      "logps/rejected": -70.11753845214844,
      "loss": 14.0276,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.013672858476638794,
      "rewards/margins": -0.013126525096595287,
      "rewards/rejected": -0.0005463305860757828,
      "step": 150
    },
    {
      "epoch": 0.08770401347505373,
      "grad_norm": 385.97100830078125,
      "learning_rate": 4.780650784427659e-06,
      "logits/chosen": -0.974000096321106,
      "logits/rejected": -0.8754386901855469,
      "logps/chosen": -78.10624694824219,
      "logps/rejected": -71.39083099365234,
      "loss": 13.8936,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005910863634198904,
      "rewards/margins": -0.0005791831645183265,
      "rewards/rejected": 0.006490050349384546,
      "step": 151
    },
    {
      "epoch": 0.08828483475634548,
      "grad_norm": 288.5458679199219,
      "learning_rate": 4.779198140615921e-06,
      "logits/chosen": -0.8864970207214355,
      "logits/rejected": -0.8748113512992859,
      "logps/chosen": -74.03097534179688,
      "logps/rejected": -68.74441528320312,
      "loss": 13.8866,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.009058129973709583,
      "rewards/margins": 0.003120752517133951,
      "rewards/rejected": 0.005937379319220781,
      "step": 152
    },
    {
      "epoch": 0.08886565603763721,
      "grad_norm": 335.7358093261719,
      "learning_rate": 4.777745496804184e-06,
      "logits/chosen": -0.7208220362663269,
      "logits/rejected": -0.7323756217956543,
      "logps/chosen": -71.64644622802734,
      "logps/rejected": -80.21810913085938,
      "loss": 13.6955,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.004168490879237652,
      "rewards/margins": 0.020404014736413956,
      "rewards/rejected": -0.01623552106320858,
      "step": 153
    },
    {
      "epoch": 0.08944647731892896,
      "grad_norm": 302.51763916015625,
      "learning_rate": 4.776292852992447e-06,
      "logits/chosen": -0.6920525431632996,
      "logits/rejected": -0.7309268712997437,
      "logps/chosen": -69.67924499511719,
      "logps/rejected": -76.42180633544922,
      "loss": 13.5794,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.005118966568261385,
      "rewards/margins": 0.031105687841773033,
      "rewards/rejected": -0.03622465208172798,
      "step": 154
    },
    {
      "epoch": 0.09002729860022071,
      "grad_norm": 316.7055358886719,
      "learning_rate": 4.7748402091807095e-06,
      "logits/chosen": -0.7559612989425659,
      "logits/rejected": -0.6078428626060486,
      "logps/chosen": -70.20340728759766,
      "logps/rejected": -66.18345642089844,
      "loss": 13.9598,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0274501983076334,
      "rewards/margins": -0.007842998020350933,
      "rewards/rejected": -0.019607199355959892,
      "step": 155
    },
    {
      "epoch": 0.09060811988151246,
      "grad_norm": 328.9933166503906,
      "learning_rate": 4.773387565368972e-06,
      "logits/chosen": -0.7114741802215576,
      "logits/rejected": -0.8936127424240112,
      "logps/chosen": -72.7677993774414,
      "logps/rejected": -77.62763214111328,
      "loss": 13.5946,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.008942861109972,
      "rewards/margins": 0.030701924115419388,
      "rewards/rejected": -0.03964478522539139,
      "step": 156
    },
    {
      "epoch": 0.09118894116280421,
      "grad_norm": 337.06060791015625,
      "learning_rate": 4.771934921557235e-06,
      "logits/chosen": -0.86445152759552,
      "logits/rejected": -0.8468233942985535,
      "logps/chosen": -84.74078369140625,
      "logps/rejected": -83.00550079345703,
      "loss": 13.9382,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03650820627808571,
      "rewards/margins": -0.002918243408203125,
      "rewards/rejected": -0.03358996659517288,
      "step": 157
    },
    {
      "epoch": 0.09176976244409595,
      "grad_norm": 318.7173767089844,
      "learning_rate": 4.770482277745498e-06,
      "logits/chosen": -0.7902041673660278,
      "logits/rejected": -0.8102203607559204,
      "logps/chosen": -72.93890380859375,
      "logps/rejected": -77.42249298095703,
      "loss": 13.745,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01063698623329401,
      "rewards/margins": 0.013827304355800152,
      "rewards/rejected": -0.02446429245173931,
      "step": 158
    },
    {
      "epoch": 0.0923505837253877,
      "grad_norm": 347.96240234375,
      "learning_rate": 4.76902963393376e-06,
      "logits/chosen": -0.8171346783638,
      "logits/rejected": -0.8012442588806152,
      "logps/chosen": -69.07205963134766,
      "logps/rejected": -80.80284118652344,
      "loss": 13.7243,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05406276509165764,
      "rewards/margins": 0.016692936420440674,
      "rewards/rejected": -0.07075570523738861,
      "step": 159
    },
    {
      "epoch": 0.09293140500667944,
      "grad_norm": 312.94622802734375,
      "learning_rate": 4.767576990122022e-06,
      "logits/chosen": -0.8780553936958313,
      "logits/rejected": -0.7555993795394897,
      "logps/chosen": -75.97615814208984,
      "logps/rejected": -73.26390838623047,
      "loss": 13.8975,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01142085064202547,
      "rewards/margins": -0.0006685241824015975,
      "rewards/rejected": -0.010752325877547264,
      "step": 160
    },
    {
      "epoch": 0.09351222628797119,
      "grad_norm": 319.6703186035156,
      "learning_rate": 4.766124346310285e-06,
      "logits/chosen": -0.9523868560791016,
      "logits/rejected": -0.9787250757217407,
      "logps/chosen": -78.85665130615234,
      "logps/rejected": -76.03120422363281,
      "loss": 13.7211,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.00805568601936102,
      "rewards/margins": 0.019266730174422264,
      "rewards/rejected": -0.027322417125105858,
      "step": 161
    },
    {
      "epoch": 0.09409304756926294,
      "grad_norm": 304.69439697265625,
      "learning_rate": 4.764671702498548e-06,
      "logits/chosen": -0.7667674422264099,
      "logits/rejected": -0.8059636354446411,
      "logps/chosen": -71.68733215332031,
      "logps/rejected": -74.61915588378906,
      "loss": 13.3629,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.03330715745687485,
      "rewards/margins": 0.05361800268292427,
      "rewards/rejected": -0.02031084895133972,
      "step": 162
    },
    {
      "epoch": 0.09467386885055469,
      "grad_norm": 302.20050048828125,
      "learning_rate": 4.763219058686811e-06,
      "logits/chosen": -0.7947182655334473,
      "logits/rejected": -0.8342369198799133,
      "logps/chosen": -71.64237213134766,
      "logps/rejected": -70.30358123779297,
      "loss": 13.4606,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.010867975652217865,
      "rewards/margins": 0.04301146790385246,
      "rewards/rejected": -0.053879447281360626,
      "step": 163
    },
    {
      "epoch": 0.09525469013184643,
      "grad_norm": 289.6075134277344,
      "learning_rate": 4.761766414875073e-06,
      "logits/chosen": -0.7613986134529114,
      "logits/rejected": -0.8836091160774231,
      "logps/chosen": -67.74214935302734,
      "logps/rejected": -78.71183776855469,
      "loss": 13.2522,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0031068138778209686,
      "rewards/margins": 0.06571229547262192,
      "rewards/rejected": -0.06881911307573318,
      "step": 164
    },
    {
      "epoch": 0.09583551141313817,
      "grad_norm": 326.1537780761719,
      "learning_rate": 4.760313771063336e-06,
      "logits/chosen": -0.8384913206100464,
      "logits/rejected": -0.7940191030502319,
      "logps/chosen": -66.64979553222656,
      "logps/rejected": -69.57666015625,
      "loss": 13.5813,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01473635621368885,
      "rewards/margins": 0.02965986728668213,
      "rewards/rejected": -0.014923503622412682,
      "step": 165
    },
    {
      "epoch": 0.09641633269442992,
      "grad_norm": 317.4963684082031,
      "learning_rate": 4.758861127251598e-06,
      "logits/chosen": -0.7010576128959656,
      "logits/rejected": -0.7300230860710144,
      "logps/chosen": -76.84666442871094,
      "logps/rejected": -84.0490493774414,
      "loss": 13.9696,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04205438122153282,
      "rewards/margins": -0.005721705500036478,
      "rewards/rejected": -0.03633267432451248,
      "step": 166
    },
    {
      "epoch": 0.09699715397572167,
      "grad_norm": 318.7678527832031,
      "learning_rate": 4.757408483439861e-06,
      "logits/chosen": -0.9276505708694458,
      "logits/rejected": -0.9561537504196167,
      "logps/chosen": -71.64440155029297,
      "logps/rejected": -80.91627502441406,
      "loss": 13.4072,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0023384944070130587,
      "rewards/margins": 0.04981740936636925,
      "rewards/rejected": -0.05215590447187424,
      "step": 167
    },
    {
      "epoch": 0.09757797525701342,
      "grad_norm": 287.4025573730469,
      "learning_rate": 4.7559558396281235e-06,
      "logits/chosen": -0.7570281624794006,
      "logits/rejected": -0.8512780070304871,
      "logps/chosen": -68.49506378173828,
      "logps/rejected": -70.23898315429688,
      "loss": 13.8651,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.007149294018745422,
      "rewards/margins": 0.0015381794655695558,
      "rewards/rejected": -0.008687476627528667,
      "step": 168
    },
    {
      "epoch": 0.09815879653830516,
      "grad_norm": 310.3047790527344,
      "learning_rate": 4.754503195816386e-06,
      "logits/chosen": -0.8154839277267456,
      "logits/rejected": -1.0209014415740967,
      "logps/chosen": -78.32124328613281,
      "logps/rejected": -68.3904037475586,
      "loss": 13.368,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.016737347468733788,
      "rewards/margins": 0.0557125024497509,
      "rewards/rejected": -0.038975149393081665,
      "step": 169
    },
    {
      "epoch": 0.0987396178195969,
      "grad_norm": 303.63385009765625,
      "learning_rate": 4.753050552004649e-06,
      "logits/chosen": -0.7536464333534241,
      "logits/rejected": -0.7967413663864136,
      "logps/chosen": -73.55670928955078,
      "logps/rejected": -75.25071716308594,
      "loss": 13.9022,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.022630508989095688,
      "rewards/margins": -0.0017447940772399306,
      "rewards/rejected": -0.020885715261101723,
      "step": 170
    },
    {
      "epoch": 0.09932043910088866,
      "grad_norm": 328.81494140625,
      "learning_rate": 4.751597908192912e-06,
      "logits/chosen": -0.9858807325363159,
      "logits/rejected": -0.9747235178947449,
      "logps/chosen": -79.53309631347656,
      "logps/rejected": -64.26655578613281,
      "loss": 14.1977,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03506975993514061,
      "rewards/margins": -0.029723864048719406,
      "rewards/rejected": -0.005345895420759916,
      "step": 171
    },
    {
      "epoch": 0.0999012603821804,
      "grad_norm": 303.1661682128906,
      "learning_rate": 4.750145264381174e-06,
      "logits/chosen": -0.8304456472396851,
      "logits/rejected": -0.754758358001709,
      "logps/chosen": -79.57183837890625,
      "logps/rejected": -73.25392150878906,
      "loss": 13.3539,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0045074475929141045,
      "rewards/margins": 0.05535992234945297,
      "rewards/rejected": -0.059867363423109055,
      "step": 172
    },
    {
      "epoch": 0.10048208166347215,
      "grad_norm": 285.4842224121094,
      "learning_rate": 4.748692620569436e-06,
      "logits/chosen": -0.7988349199295044,
      "logits/rejected": -0.8758390545845032,
      "logps/chosen": -69.04563903808594,
      "logps/rejected": -63.191673278808594,
      "loss": 13.8072,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02564077451825142,
      "rewards/margins": 0.009737257845699787,
      "rewards/rejected": -0.03537803143262863,
      "step": 173
    },
    {
      "epoch": 0.1010629029447639,
      "grad_norm": 316.76220703125,
      "learning_rate": 4.747239976757699e-06,
      "logits/chosen": -0.8752381205558777,
      "logits/rejected": -0.9098547697067261,
      "logps/chosen": -71.7885513305664,
      "logps/rejected": -77.3386001586914,
      "loss": 13.8798,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03445696085691452,
      "rewards/margins": 0.00015344536222983152,
      "rewards/rejected": -0.03461039811372757,
      "step": 174
    },
    {
      "epoch": 0.10164372422605564,
      "grad_norm": 285.7683410644531,
      "learning_rate": 4.745787332945962e-06,
      "logits/chosen": -0.8894672393798828,
      "logits/rejected": -1.066870927810669,
      "logps/chosen": -71.97786712646484,
      "logps/rejected": -70.2954330444336,
      "loss": 13.8723,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.027886096388101578,
      "rewards/margins": 0.003271915018558502,
      "rewards/rejected": -0.03115800954401493,
      "step": 175
    },
    {
      "epoch": 0.10222454550734739,
      "grad_norm": 314.2114562988281,
      "learning_rate": 4.744334689134225e-06,
      "logits/chosen": -0.9301921725273132,
      "logits/rejected": -0.9640763401985168,
      "logps/chosen": -68.97576904296875,
      "logps/rejected": -73.05828857421875,
      "loss": 13.546,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.00339788431301713,
      "rewards/margins": 0.03368956595659256,
      "rewards/rejected": -0.03708745166659355,
      "step": 176
    },
    {
      "epoch": 0.10280536678863914,
      "grad_norm": 350.662353515625,
      "learning_rate": 4.742882045322487e-06,
      "logits/chosen": -0.9687705039978027,
      "logits/rejected": -0.8688680529594421,
      "logps/chosen": -76.81888580322266,
      "logps/rejected": -75.65296936035156,
      "loss": 14.2808,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07494613528251648,
      "rewards/margins": -0.03318053483963013,
      "rewards/rejected": -0.04176560416817665,
      "step": 177
    },
    {
      "epoch": 0.10338618806993088,
      "grad_norm": 335.7806396484375,
      "learning_rate": 4.74142940151075e-06,
      "logits/chosen": -0.7820795178413391,
      "logits/rejected": -0.8843638300895691,
      "logps/chosen": -83.46534729003906,
      "logps/rejected": -73.49069213867188,
      "loss": 13.5938,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.011855507269501686,
      "rewards/margins": 0.030727148056030273,
      "rewards/rejected": -0.04258265346288681,
      "step": 178
    },
    {
      "epoch": 0.10396700935122263,
      "grad_norm": 298.5807800292969,
      "learning_rate": 4.739976757699012e-06,
      "logits/chosen": -0.9966332316398621,
      "logits/rejected": -0.8384010195732117,
      "logps/chosen": -75.45916748046875,
      "logps/rejected": -79.22760009765625,
      "loss": 13.5549,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.045658472925424576,
      "rewards/margins": 0.03411347419023514,
      "rewards/rejected": -0.07977195084095001,
      "step": 179
    },
    {
      "epoch": 0.10454783063251437,
      "grad_norm": 308.5312805175781,
      "learning_rate": 4.738524113887275e-06,
      "logits/chosen": -1.1542797088623047,
      "logits/rejected": -1.1540412902832031,
      "logps/chosen": -71.09564971923828,
      "logps/rejected": -66.72710418701172,
      "loss": 13.8693,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01030341349542141,
      "rewards/margins": 0.0030408282764256,
      "rewards/rejected": -0.013344240374863148,
      "step": 180
    },
    {
      "epoch": 0.10512865191380612,
      "grad_norm": 382.357666015625,
      "learning_rate": 4.7370714700755375e-06,
      "logits/chosen": -0.8503016233444214,
      "logits/rejected": -0.8578587770462036,
      "logps/chosen": -67.1768798828125,
      "logps/rejected": -74.30778503417969,
      "loss": 13.5419,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024283096194267273,
      "rewards/margins": 0.036231573671102524,
      "rewards/rejected": -0.0605146698653698,
      "step": 181
    },
    {
      "epoch": 0.10570947319509787,
      "grad_norm": 316.55859375,
      "learning_rate": 4.7356188262638e-06,
      "logits/chosen": -0.8488529324531555,
      "logits/rejected": -0.8070418238639832,
      "logps/chosen": -75.79255676269531,
      "logps/rejected": -76.4787826538086,
      "loss": 13.6775,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01909615471959114,
      "rewards/margins": 0.02385520003736019,
      "rewards/rejected": -0.04295135289430618,
      "step": 182
    },
    {
      "epoch": 0.10629029447638962,
      "grad_norm": 309.40887451171875,
      "learning_rate": 4.734166182452063e-06,
      "logits/chosen": -0.9303333163261414,
      "logits/rejected": -0.900216281414032,
      "logps/chosen": -74.81334686279297,
      "logps/rejected": -76.99653625488281,
      "loss": 14.0831,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.08230753988027573,
      "rewards/margins": -0.01739531196653843,
      "rewards/rejected": -0.06491222977638245,
      "step": 183
    },
    {
      "epoch": 0.10687111575768136,
      "grad_norm": 308.8415832519531,
      "learning_rate": 4.732713538640326e-06,
      "logits/chosen": -0.9111288785934448,
      "logits/rejected": -0.8835655450820923,
      "logps/chosen": -82.20323181152344,
      "logps/rejected": -69.86083221435547,
      "loss": 13.6755,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03169438987970352,
      "rewards/margins": 0.023478079587221146,
      "rewards/rejected": -0.055172473192214966,
      "step": 184
    },
    {
      "epoch": 0.10745193703897311,
      "grad_norm": 294.7135925292969,
      "learning_rate": 4.7312608948285885e-06,
      "logits/chosen": -0.7559576630592346,
      "logits/rejected": -0.7548641562461853,
      "logps/chosen": -72.80931091308594,
      "logps/rejected": -68.47846221923828,
      "loss": 13.6632,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03249110281467438,
      "rewards/margins": 0.026273757219314575,
      "rewards/rejected": -0.05876486748456955,
      "step": 185
    },
    {
      "epoch": 0.10803275832026485,
      "grad_norm": 300.0313720703125,
      "learning_rate": 4.7298082510168504e-06,
      "logits/chosen": -0.7012637853622437,
      "logits/rejected": -0.7670835256576538,
      "logps/chosen": -69.26997375488281,
      "logps/rejected": -75.66789245605469,
      "loss": 13.8065,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05606495216488838,
      "rewards/margins": 0.008019248023629189,
      "rewards/rejected": -0.06408419460058212,
      "step": 186
    },
    {
      "epoch": 0.1086135796015566,
      "grad_norm": 305.9514465332031,
      "learning_rate": 4.728355607205113e-06,
      "logits/chosen": -0.848249614238739,
      "logits/rejected": -0.9618538022041321,
      "logps/chosen": -76.71751403808594,
      "logps/rejected": -75.86561584472656,
      "loss": 13.9142,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.058838047087192535,
      "rewards/margins": -0.0012072951067239046,
      "rewards/rejected": -0.057630755007267,
      "step": 187
    },
    {
      "epoch": 0.10919440088284835,
      "grad_norm": 299.03106689453125,
      "learning_rate": 4.726902963393376e-06,
      "logits/chosen": -0.7864362001419067,
      "logits/rejected": -0.9607146382331848,
      "logps/chosen": -73.52447509765625,
      "logps/rejected": -83.42733001708984,
      "loss": 13.0054,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.0022229477763175964,
      "rewards/margins": 0.09174972772598267,
      "rewards/rejected": -0.08952677994966507,
      "step": 188
    },
    {
      "epoch": 0.1097752221641401,
      "grad_norm": 301.38275146484375,
      "learning_rate": 4.7254503195816395e-06,
      "logits/chosen": -0.9863910675048828,
      "logits/rejected": -0.9701216816902161,
      "logps/chosen": -73.6039047241211,
      "logps/rejected": -74.17659759521484,
      "loss": 13.5522,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.027939602732658386,
      "rewards/margins": 0.033790357410907745,
      "rewards/rejected": -0.06172995641827583,
      "step": 189
    },
    {
      "epoch": 0.11035604344543185,
      "grad_norm": 321.6877136230469,
      "learning_rate": 4.723997675769902e-06,
      "logits/chosen": -0.796249508857727,
      "logits/rejected": -0.7661502957344055,
      "logps/chosen": -78.24516296386719,
      "logps/rejected": -74.38370513916016,
      "loss": 13.5199,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.038561441004276276,
      "rewards/margins": 0.039519913494586945,
      "rewards/rejected": -0.07808135449886322,
      "step": 190
    },
    {
      "epoch": 0.11093686472672358,
      "grad_norm": 317.3888854980469,
      "learning_rate": 4.722545031958164e-06,
      "logits/chosen": -0.9007613062858582,
      "logits/rejected": -0.9852052927017212,
      "logps/chosen": -82.073486328125,
      "logps/rejected": -72.91219329833984,
      "loss": 13.6764,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03767850995063782,
      "rewards/margins": 0.023520758375525475,
      "rewards/rejected": -0.06119927018880844,
      "step": 191
    },
    {
      "epoch": 0.11151768600801533,
      "grad_norm": 316.7236328125,
      "learning_rate": 4.721092388146427e-06,
      "logits/chosen": -0.983284592628479,
      "logits/rejected": -0.8711267709732056,
      "logps/chosen": -76.91285705566406,
      "logps/rejected": -67.65292358398438,
      "loss": 14.1306,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03306933492422104,
      "rewards/margins": -0.022902240976691246,
      "rewards/rejected": -0.010167093947529793,
      "step": 192
    },
    {
      "epoch": 0.11209850728930708,
      "grad_norm": 313.7123718261719,
      "learning_rate": 4.71963974433469e-06,
      "logits/chosen": -0.9627790451049805,
      "logits/rejected": -0.8592319488525391,
      "logps/chosen": -76.01704406738281,
      "logps/rejected": -72.71672058105469,
      "loss": 14.0736,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.049292050302028656,
      "rewards/margins": -0.01814861036837101,
      "rewards/rejected": -0.031143436208367348,
      "step": 193
    },
    {
      "epoch": 0.11267932857059883,
      "grad_norm": 303.0953063964844,
      "learning_rate": 4.718187100522952e-06,
      "logits/chosen": -0.8993155360221863,
      "logits/rejected": -0.8849833607673645,
      "logps/chosen": -71.47299194335938,
      "logps/rejected": -75.4497299194336,
      "loss": 13.3576,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004056071862578392,
      "rewards/margins": 0.05458641052246094,
      "rewards/rejected": -0.050530336797237396,
      "step": 194
    },
    {
      "epoch": 0.11326014985189058,
      "grad_norm": 311.2488098144531,
      "learning_rate": 4.716734456711215e-06,
      "logits/chosen": -0.9742434620857239,
      "logits/rejected": -0.9584578275680542,
      "logps/chosen": -75.96842956542969,
      "logps/rejected": -71.90623474121094,
      "loss": 13.6582,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01095439214259386,
      "rewards/margins": 0.023807067424058914,
      "rewards/rejected": -0.0347614586353302,
      "step": 195
    },
    {
      "epoch": 0.11384097113318233,
      "grad_norm": 300.0875549316406,
      "learning_rate": 4.715281812899478e-06,
      "logits/chosen": -0.594025194644928,
      "logits/rejected": -0.6777793765068054,
      "logps/chosen": -74.22691345214844,
      "logps/rejected": -74.91129302978516,
      "loss": 13.5482,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0018765062559396029,
      "rewards/margins": 0.03627584129571915,
      "rewards/rejected": -0.03439933806657791,
      "step": 196
    },
    {
      "epoch": 0.11442179241447406,
      "grad_norm": 299.1294860839844,
      "learning_rate": 4.713829169087741e-06,
      "logits/chosen": -0.7969453930854797,
      "logits/rejected": -0.8590051531791687,
      "logps/chosen": -74.24263000488281,
      "logps/rejected": -80.28329467773438,
      "loss": 13.9374,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.029978638514876366,
      "rewards/margins": -0.0027882575523108244,
      "rewards/rejected": -0.027190377935767174,
      "step": 197
    },
    {
      "epoch": 0.11500261369576581,
      "grad_norm": 320.9808044433594,
      "learning_rate": 4.7123765252760026e-06,
      "logits/chosen": -0.9466894865036011,
      "logits/rejected": -1.0433050394058228,
      "logps/chosen": -72.66864776611328,
      "logps/rejected": -75.08808135986328,
      "loss": 14.3219,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.033920880407094955,
      "rewards/margins": -0.04234874248504639,
      "rewards/rejected": 0.008427867665886879,
      "step": 198
    },
    {
      "epoch": 0.11558343497705756,
      "grad_norm": 308.2682189941406,
      "learning_rate": 4.710923881464265e-06,
      "logits/chosen": -0.7543210387229919,
      "logits/rejected": -0.8157938718795776,
      "logps/chosen": -81.17222595214844,
      "logps/rejected": -72.63612365722656,
      "loss": 14.0541,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.022938327863812447,
      "rewards/margins": -0.015443983487784863,
      "rewards/rejected": -0.007494345307350159,
      "step": 199
    },
    {
      "epoch": 0.11616425625834931,
      "grad_norm": 312.9922180175781,
      "learning_rate": 4.709471237652528e-06,
      "logits/chosen": -0.6948032975196838,
      "logits/rejected": -0.6942026615142822,
      "logps/chosen": -73.12284851074219,
      "logps/rejected": -77.82487487792969,
      "loss": 13.9199,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.029393082484602928,
      "rewards/margins": -0.0029410452116280794,
      "rewards/rejected": -0.02645203471183777,
      "step": 200
    },
    {
      "epoch": 0.11674507753964106,
      "grad_norm": 311.9326477050781,
      "learning_rate": 4.708018593840791e-06,
      "logits/chosen": -1.1453566551208496,
      "logits/rejected": -1.1129220724105835,
      "logps/chosen": -68.6366958618164,
      "logps/rejected": -73.55378723144531,
      "loss": 13.8228,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.010440112091600895,
      "rewards/margins": 0.009912310168147087,
      "rewards/rejected": -0.020352421328425407,
      "step": 201
    },
    {
      "epoch": 0.1173258988209328,
      "grad_norm": 308.12060546875,
      "learning_rate": 4.7065659500290536e-06,
      "logits/chosen": -0.8134487271308899,
      "logits/rejected": -0.7991577386856079,
      "logps/chosen": -70.990966796875,
      "logps/rejected": -78.31260681152344,
      "loss": 13.2819,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.004076024051755667,
      "rewards/margins": 0.06265188753604889,
      "rewards/rejected": -0.06672791391611099,
      "step": 202
    },
    {
      "epoch": 0.11790672010222454,
      "grad_norm": 287.57794189453125,
      "learning_rate": 4.705113306217316e-06,
      "logits/chosen": -0.9393211603164673,
      "logits/rejected": -0.9954279065132141,
      "logps/chosen": -76.2332534790039,
      "logps/rejected": -70.47115325927734,
      "loss": 13.2255,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.00021018953702878207,
      "rewards/margins": 0.06832081824541092,
      "rewards/rejected": -0.06853101402521133,
      "step": 203
    },
    {
      "epoch": 0.11848754138351629,
      "grad_norm": 305.3521423339844,
      "learning_rate": 4.703660662405578e-06,
      "logits/chosen": -0.9575725793838501,
      "logits/rejected": -1.0199588537216187,
      "logps/chosen": -80.7159423828125,
      "logps/rejected": -67.89506530761719,
      "loss": 14.0192,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.06569384038448334,
      "rewards/margins": -0.01254147756844759,
      "rewards/rejected": -0.053152360022068024,
      "step": 204
    },
    {
      "epoch": 0.11906836266480804,
      "grad_norm": 278.6866760253906,
      "learning_rate": 4.702208018593841e-06,
      "logits/chosen": -0.7608988881111145,
      "logits/rejected": -0.7628680467605591,
      "logps/chosen": -70.3031005859375,
      "logps/rejected": -70.50173950195312,
      "loss": 13.39,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.021148493513464928,
      "rewards/margins": 0.052839674055576324,
      "rewards/rejected": -0.0739881694316864,
      "step": 205
    },
    {
      "epoch": 0.11964918394609979,
      "grad_norm": 304.92913818359375,
      "learning_rate": 4.700755374782104e-06,
      "logits/chosen": -0.8696325421333313,
      "logits/rejected": -0.9427449107170105,
      "logps/chosen": -74.74107360839844,
      "logps/rejected": -75.46221160888672,
      "loss": 13.7106,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.051121801137924194,
      "rewards/margins": 0.018347539007663727,
      "rewards/rejected": -0.06946934014558792,
      "step": 206
    },
    {
      "epoch": 0.12023000522739154,
      "grad_norm": 315.7526550292969,
      "learning_rate": 4.6993027309703665e-06,
      "logits/chosen": -0.8430768251419067,
      "logits/rejected": -0.7843033075332642,
      "logps/chosen": -71.7156982421875,
      "logps/rejected": -74.4084243774414,
      "loss": 13.8928,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.024820514023303986,
      "rewards/margins": -0.0010401479667052627,
      "rewards/rejected": -0.023780368268489838,
      "step": 207
    },
    {
      "epoch": 0.12081082650868327,
      "grad_norm": 380.44561767578125,
      "learning_rate": 4.697850087158629e-06,
      "logits/chosen": -0.8150386810302734,
      "logits/rejected": -0.7826833724975586,
      "logps/chosen": -76.09224700927734,
      "logps/rejected": -83.32121276855469,
      "loss": 13.609,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.032852984964847565,
      "rewards/margins": 0.02871485985815525,
      "rewards/rejected": -0.061567842960357666,
      "step": 208
    },
    {
      "epoch": 0.12139164778997502,
      "grad_norm": 330.4120788574219,
      "learning_rate": 4.696397443346892e-06,
      "logits/chosen": -1.0164332389831543,
      "logits/rejected": -0.9415372610092163,
      "logps/chosen": -78.38670349121094,
      "logps/rejected": -79.13145446777344,
      "loss": 14.0359,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.061431754380464554,
      "rewards/margins": -0.01327196042984724,
      "rewards/rejected": -0.04815979301929474,
      "step": 209
    },
    {
      "epoch": 0.12197246907126677,
      "grad_norm": 336.34759521484375,
      "learning_rate": 4.694944799535155e-06,
      "logits/chosen": -0.9332435727119446,
      "logits/rejected": -0.9620237350463867,
      "logps/chosen": -78.70216369628906,
      "logps/rejected": -90.57441711425781,
      "loss": 13.5176,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.00978778675198555,
      "rewards/margins": 0.0402628593146801,
      "rewards/rejected": -0.050050653517246246,
      "step": 210
    },
    {
      "epoch": 0.12255329035255852,
      "grad_norm": 303.8833923339844,
      "learning_rate": 4.693492155723417e-06,
      "logits/chosen": -0.9775910377502441,
      "logits/rejected": -0.9736326932907104,
      "logps/chosen": -78.42801666259766,
      "logps/rejected": -73.2563705444336,
      "loss": 14.1172,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.021739017218351364,
      "rewards/margins": -0.02232358045876026,
      "rewards/rejected": 0.0005845635896548629,
      "step": 211
    },
    {
      "epoch": 0.12313411163385027,
      "grad_norm": 307.4227294921875,
      "learning_rate": 4.692039511911679e-06,
      "logits/chosen": -0.8911228179931641,
      "logits/rejected": -1.0791237354278564,
      "logps/chosen": -65.61485290527344,
      "logps/rejected": -75.7813720703125,
      "loss": 13.3959,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.014038696885108948,
      "rewards/margins": 0.05425255373120308,
      "rewards/rejected": -0.04021385312080383,
      "step": 212
    },
    {
      "epoch": 0.12371493291514202,
      "grad_norm": 287.40045166015625,
      "learning_rate": 4.690586868099942e-06,
      "logits/chosen": -0.8995906710624695,
      "logits/rejected": -0.8471341133117676,
      "logps/chosen": -66.9594497680664,
      "logps/rejected": -71.04551696777344,
      "loss": 13.6394,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01811062917113304,
      "rewards/margins": 0.027448922395706177,
      "rewards/rejected": -0.04555954784154892,
      "step": 213
    },
    {
      "epoch": 0.12429575419643375,
      "grad_norm": 306.50128173828125,
      "learning_rate": 4.689134224288205e-06,
      "logits/chosen": -0.8109323382377625,
      "logits/rejected": -0.7679190635681152,
      "logps/chosen": -66.3537826538086,
      "logps/rejected": -78.93473815917969,
      "loss": 13.4859,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04216752201318741,
      "rewards/margins": 0.0444951057434082,
      "rewards/rejected": -0.08666262775659561,
      "step": 214
    },
    {
      "epoch": 0.1248765754777255,
      "grad_norm": 327.2640075683594,
      "learning_rate": 4.687681580476468e-06,
      "logits/chosen": -0.895319938659668,
      "logits/rejected": -0.9245772361755371,
      "logps/chosen": -69.65949249267578,
      "logps/rejected": -71.36875915527344,
      "loss": 14.5911,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.08888588845729828,
      "rewards/margins": -0.06758525222539902,
      "rewards/rejected": -0.021300649270415306,
      "step": 215
    },
    {
      "epoch": 0.12545739675901724,
      "grad_norm": 322.3681640625,
      "learning_rate": 4.68622893666473e-06,
      "logits/chosen": -0.8954951167106628,
      "logits/rejected": -0.81683349609375,
      "logps/chosen": -82.0992431640625,
      "logps/rejected": -71.04183197021484,
      "loss": 14.0854,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0422581322491169,
      "rewards/margins": -0.01680077239871025,
      "rewards/rejected": -0.025457357987761497,
      "step": 216
    },
    {
      "epoch": 0.12603821804030899,
      "grad_norm": 332.36920166015625,
      "learning_rate": 4.684776292852993e-06,
      "logits/chosen": -0.887475311756134,
      "logits/rejected": -0.7960731983184814,
      "logps/chosen": -74.29244995117188,
      "logps/rejected": -81.73658752441406,
      "loss": 14.184,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.09383013099431992,
      "rewards/margins": -0.027784889563918114,
      "rewards/rejected": -0.06604524701833725,
      "step": 217
    },
    {
      "epoch": 0.12661903932160073,
      "grad_norm": 329.1742858886719,
      "learning_rate": 4.683323649041255e-06,
      "logits/chosen": -0.7891820073127747,
      "logits/rejected": -1.006798505783081,
      "logps/chosen": -77.01458740234375,
      "logps/rejected": -71.24549102783203,
      "loss": 14.3551,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.07353498041629791,
      "rewards/margins": -0.045538146048784256,
      "rewards/rejected": -0.02799682877957821,
      "step": 218
    },
    {
      "epoch": 0.12719986060289248,
      "grad_norm": 300.3213806152344,
      "learning_rate": 4.681871005229518e-06,
      "logits/chosen": -0.9636018872261047,
      "logits/rejected": -1.0088518857955933,
      "logps/chosen": -64.98380279541016,
      "logps/rejected": -71.98371887207031,
      "loss": 13.5981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009727184660732746,
      "rewards/margins": 0.030616506934165955,
      "rewards/rejected": -0.04034368693828583,
      "step": 219
    },
    {
      "epoch": 0.12778068188418423,
      "grad_norm": 316.0754699707031,
      "learning_rate": 4.6804183614177805e-06,
      "logits/chosen": -0.8850613832473755,
      "logits/rejected": -0.9165294766426086,
      "logps/chosen": -77.82759857177734,
      "logps/rejected": -73.77207946777344,
      "loss": 13.467,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.020776253193616867,
      "rewards/margins": 0.04324489086866379,
      "rewards/rejected": -0.022468645125627518,
      "step": 220
    },
    {
      "epoch": 0.12836150316547598,
      "grad_norm": 298.15130615234375,
      "learning_rate": 4.678965717606043e-06,
      "logits/chosen": -0.9083736538887024,
      "logits/rejected": -0.9166957139968872,
      "logps/chosen": -64.89097595214844,
      "logps/rejected": -71.65401458740234,
      "loss": 13.4666,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.004524010233581066,
      "rewards/margins": 0.045348040759563446,
      "rewards/rejected": -0.049872055649757385,
      "step": 221
    },
    {
      "epoch": 0.12894232444676773,
      "grad_norm": 328.2380065917969,
      "learning_rate": 4.677513073794306e-06,
      "logits/chosen": -0.8878594636917114,
      "logits/rejected": -0.8982254862785339,
      "logps/chosen": -74.40402221679688,
      "logps/rejected": -67.42572021484375,
      "loss": 14.4278,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.07353280484676361,
      "rewards/margins": -0.05194491147994995,
      "rewards/rejected": -0.02158789522945881,
      "step": 222
    },
    {
      "epoch": 0.12952314572805948,
      "grad_norm": 312.8938903808594,
      "learning_rate": 4.676060429982569e-06,
      "logits/chosen": -0.9499589204788208,
      "logits/rejected": -0.9511539340019226,
      "logps/chosen": -74.2853012084961,
      "logps/rejected": -74.38381958007812,
      "loss": 13.8183,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04615163058042526,
      "rewards/margins": 0.010002289898693562,
      "rewards/rejected": -0.05615391582250595,
      "step": 223
    },
    {
      "epoch": 0.13010396700935123,
      "grad_norm": 308.1676330566406,
      "learning_rate": 4.6746077861708315e-06,
      "logits/chosen": -0.9691025614738464,
      "logits/rejected": -1.0638432502746582,
      "logps/chosen": -68.75132751464844,
      "logps/rejected": -64.51249694824219,
      "loss": 14.5132,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.0783403068780899,
      "rewards/margins": -0.058567456901073456,
      "rewards/rejected": -0.019772853702306747,
      "step": 224
    },
    {
      "epoch": 0.13068478829064298,
      "grad_norm": 300.42218017578125,
      "learning_rate": 4.673155142359093e-06,
      "logits/chosen": -0.9048269391059875,
      "logits/rejected": -0.9264401197433472,
      "logps/chosen": -73.9933090209961,
      "logps/rejected": -80.50711822509766,
      "loss": 13.5877,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.017903203144669533,
      "rewards/margins": 0.03073006495833397,
      "rewards/rejected": -0.04863326996564865,
      "step": 225
    },
    {
      "epoch": 0.13126560957193473,
      "grad_norm": 344.2493896484375,
      "learning_rate": 4.671702498547356e-06,
      "logits/chosen": -0.8985874056816101,
      "logits/rejected": -0.843769371509552,
      "logps/chosen": -72.87957000732422,
      "logps/rejected": -78.47367095947266,
      "loss": 13.6586,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01365675963461399,
      "rewards/margins": 0.0262027345597744,
      "rewards/rejected": -0.03985949605703354,
      "step": 226
    },
    {
      "epoch": 0.13184643085322645,
      "grad_norm": 314.46856689453125,
      "learning_rate": 4.670249854735619e-06,
      "logits/chosen": -0.9261584281921387,
      "logits/rejected": -1.041046142578125,
      "logps/chosen": -76.67170715332031,
      "logps/rejected": -66.8155517578125,
      "loss": 13.9811,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03505679965019226,
      "rewards/margins": -0.007680593524128199,
      "rewards/rejected": -0.0273762084543705,
      "step": 227
    },
    {
      "epoch": 0.1324272521345182,
      "grad_norm": 341.5300598144531,
      "learning_rate": 4.668797210923882e-06,
      "logits/chosen": -0.8199512362480164,
      "logits/rejected": -0.8454850316047668,
      "logps/chosen": -63.34907150268555,
      "logps/rejected": -67.3141098022461,
      "loss": 13.2625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.009090743958950043,
      "rewards/margins": 0.06578489392995834,
      "rewards/rejected": -0.0566941499710083,
      "step": 228
    },
    {
      "epoch": 0.13300807341580995,
      "grad_norm": 301.3994445800781,
      "learning_rate": 4.667344567112144e-06,
      "logits/chosen": -1.106350302696228,
      "logits/rejected": -1.0245457887649536,
      "logps/chosen": -72.74774932861328,
      "logps/rejected": -74.6384506225586,
      "loss": 14.0679,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011247700080275536,
      "rewards/margins": -0.01573079079389572,
      "rewards/rejected": 0.004483087919652462,
      "step": 229
    },
    {
      "epoch": 0.1335888946971017,
      "grad_norm": 309.1897277832031,
      "learning_rate": 4.665891923300407e-06,
      "logits/chosen": -0.9697766304016113,
      "logits/rejected": -1.008512258529663,
      "logps/chosen": -78.22880554199219,
      "logps/rejected": -62.13957977294922,
      "loss": 13.917,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.030580034479498863,
      "rewards/margins": -0.0008186303311958909,
      "rewards/rejected": -0.029761407524347305,
      "step": 230
    },
    {
      "epoch": 0.13416971597839344,
      "grad_norm": 332.3008117675781,
      "learning_rate": 4.66443927948867e-06,
      "logits/chosen": -0.9368025064468384,
      "logits/rejected": -0.8751896023750305,
      "logps/chosen": -80.72710418701172,
      "logps/rejected": -75.58308410644531,
      "loss": 13.9649,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.060019601136446,
      "rewards/margins": -0.0033089532516896725,
      "rewards/rejected": -0.05671064928174019,
      "step": 231
    },
    {
      "epoch": 0.1347505372596852,
      "grad_norm": 326.7151794433594,
      "learning_rate": 4.662986635676933e-06,
      "logits/chosen": -0.9234104156494141,
      "logits/rejected": -0.9875878095626831,
      "logps/chosen": -79.9911117553711,
      "logps/rejected": -80.50080871582031,
      "loss": 13.995,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05462227389216423,
      "rewards/margins": -0.009203016757965088,
      "rewards/rejected": -0.04541926831007004,
      "step": 232
    },
    {
      "epoch": 0.13533135854097694,
      "grad_norm": 297.00897216796875,
      "learning_rate": 4.661533991865195e-06,
      "logits/chosen": -0.6637614965438843,
      "logits/rejected": -0.8283042907714844,
      "logps/chosen": -71.94387817382812,
      "logps/rejected": -65.46293640136719,
      "loss": 14.1861,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0613144226372242,
      "rewards/margins": -0.02849789522588253,
      "rewards/rejected": -0.03281652554869652,
      "step": 233
    },
    {
      "epoch": 0.1359121798222687,
      "grad_norm": 308.1539306640625,
      "learning_rate": 4.660081348053458e-06,
      "logits/chosen": -1.0005970001220703,
      "logits/rejected": -1.005382776260376,
      "logps/chosen": -75.04243469238281,
      "logps/rejected": -70.30781555175781,
      "loss": 13.5591,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.005779562052339315,
      "rewards/margins": 0.03400464728474617,
      "rewards/rejected": -0.03978421539068222,
      "step": 234
    },
    {
      "epoch": 0.13649300110356044,
      "grad_norm": 314.4703674316406,
      "learning_rate": 4.658628704241721e-06,
      "logits/chosen": -0.8988968729972839,
      "logits/rejected": -0.9323342442512512,
      "logps/chosen": -70.89219665527344,
      "logps/rejected": -72.37486267089844,
      "loss": 13.8039,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03381185606122017,
      "rewards/margins": 0.010169675573706627,
      "rewards/rejected": -0.04398152977228165,
      "step": 235
    },
    {
      "epoch": 0.1370738223848522,
      "grad_norm": 318.1397705078125,
      "learning_rate": 4.657176060429983e-06,
      "logits/chosen": -0.9645525217056274,
      "logits/rejected": -0.8109456896781921,
      "logps/chosen": -74.8411865234375,
      "logps/rejected": -73.68878936767578,
      "loss": 14.1379,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.045103732496500015,
      "rewards/margins": -0.02444007806479931,
      "rewards/rejected": -0.020663652569055557,
      "step": 236
    },
    {
      "epoch": 0.13765464366614394,
      "grad_norm": 297.63604736328125,
      "learning_rate": 4.6557234166182455e-06,
      "logits/chosen": -0.9129088521003723,
      "logits/rejected": -0.9517717361450195,
      "logps/chosen": -68.26078033447266,
      "logps/rejected": -78.68707275390625,
      "loss": 13.3907,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.013581380248069763,
      "rewards/margins": 0.05378935858607292,
      "rewards/rejected": -0.06737073510885239,
      "step": 237
    },
    {
      "epoch": 0.1382354649474357,
      "grad_norm": 322.44964599609375,
      "learning_rate": 4.654270772806508e-06,
      "logits/chosen": -1.0076889991760254,
      "logits/rejected": -0.9823756217956543,
      "logps/chosen": -70.8648681640625,
      "logps/rejected": -75.48307037353516,
      "loss": 13.5039,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.021284928545355797,
      "rewards/margins": 0.039048969745635986,
      "rewards/rejected": -0.06033390760421753,
      "step": 238
    },
    {
      "epoch": 0.1388162862287274,
      "grad_norm": 320.17193603515625,
      "learning_rate": 4.652818128994771e-06,
      "logits/chosen": -0.9012085199356079,
      "logits/rejected": -1.0353925228118896,
      "logps/chosen": -77.64144134521484,
      "logps/rejected": -72.55008697509766,
      "loss": 13.9087,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0533623993396759,
      "rewards/margins": -0.0019609176088124514,
      "rewards/rejected": -0.05140148475766182,
      "step": 239
    },
    {
      "epoch": 0.13939710751001916,
      "grad_norm": 291.0409851074219,
      "learning_rate": 4.651365485183034e-06,
      "logits/chosen": -0.7694844603538513,
      "logits/rejected": -0.7582476735115051,
      "logps/chosen": -66.96981048583984,
      "logps/rejected": -69.53636169433594,
      "loss": 13.8481,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.029136648401618004,
      "rewards/margins": 0.006323431618511677,
      "rewards/rejected": -0.035460080951452255,
      "step": 240
    },
    {
      "epoch": 0.1399779287913109,
      "grad_norm": 309.0951843261719,
      "learning_rate": 4.6499128413712965e-06,
      "logits/chosen": -1.09603750705719,
      "logits/rejected": -1.1345731019973755,
      "logps/chosen": -69.8927001953125,
      "logps/rejected": -76.84407806396484,
      "loss": 13.2567,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.028979092836380005,
      "rewards/margins": 0.06684218347072601,
      "rewards/rejected": -0.03786309435963631,
      "step": 241
    },
    {
      "epoch": 0.14055875007260266,
      "grad_norm": 341.45458984375,
      "learning_rate": 4.648460197559559e-06,
      "logits/chosen": -0.7778705358505249,
      "logits/rejected": -0.9100780487060547,
      "logps/chosen": -83.03035736083984,
      "logps/rejected": -76.86042785644531,
      "loss": 13.7437,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.028774719685316086,
      "rewards/margins": 0.01645464077591896,
      "rewards/rejected": -0.04522935673594475,
      "step": 242
    },
    {
      "epoch": 0.1411395713538944,
      "grad_norm": 292.6492919921875,
      "learning_rate": 4.647007553747821e-06,
      "logits/chosen": -1.0126192569732666,
      "logits/rejected": -0.8217967748641968,
      "logps/chosen": -78.85697937011719,
      "logps/rejected": -69.64402770996094,
      "loss": 13.8864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.031996484845876694,
      "rewards/margins": 0.002992384135723114,
      "rewards/rejected": -0.034988872706890106,
      "step": 243
    },
    {
      "epoch": 0.14172039263518615,
      "grad_norm": 312.7855529785156,
      "learning_rate": 4.645554909936084e-06,
      "logits/chosen": -0.9456573724746704,
      "logits/rejected": -0.9571585655212402,
      "logps/chosen": -74.84808349609375,
      "logps/rejected": -71.9338150024414,
      "loss": 14.0443,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.046040650457143784,
      "rewards/margins": -0.014010493643581867,
      "rewards/rejected": -0.03203015774488449,
      "step": 244
    },
    {
      "epoch": 0.1423012139164779,
      "grad_norm": 318.9042053222656,
      "learning_rate": 4.644102266124347e-06,
      "logits/chosen": -0.7081719636917114,
      "logits/rejected": -0.8551041483879089,
      "logps/chosen": -77.06927490234375,
      "logps/rejected": -80.43268585205078,
      "loss": 13.833,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04912562295794487,
      "rewards/margins": 0.007158022373914719,
      "rewards/rejected": -0.05628364533185959,
      "step": 245
    },
    {
      "epoch": 0.14288203519776965,
      "grad_norm": 333.2130126953125,
      "learning_rate": 4.642649622312609e-06,
      "logits/chosen": -1.046338438987732,
      "logits/rejected": -1.0311440229415894,
      "logps/chosen": -76.91915893554688,
      "logps/rejected": -76.76275634765625,
      "loss": 13.7829,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.005943517200648785,
      "rewards/margins": 0.011601355858147144,
      "rewards/rejected": -0.0056578353978693485,
      "step": 246
    },
    {
      "epoch": 0.1434628564790614,
      "grad_norm": 315.692626953125,
      "learning_rate": 4.641196978500872e-06,
      "logits/chosen": -0.7881887555122375,
      "logits/rejected": -0.8423610925674438,
      "logps/chosen": -72.17633056640625,
      "logps/rejected": -80.27653503417969,
      "loss": 13.5325,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.007096032612025738,
      "rewards/margins": 0.035623110830783844,
      "rewards/rejected": -0.042719148099422455,
      "step": 247
    },
    {
      "epoch": 0.14404367776035315,
      "grad_norm": 324.49945068359375,
      "learning_rate": 4.639744334689135e-06,
      "logits/chosen": -0.9588478803634644,
      "logits/rejected": -0.9462132453918457,
      "logps/chosen": -77.13792419433594,
      "logps/rejected": -72.57328796386719,
      "loss": 13.6316,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0016011804109439254,
      "rewards/margins": 0.02814745344221592,
      "rewards/rejected": -0.02974862977862358,
      "step": 248
    },
    {
      "epoch": 0.1446244990416449,
      "grad_norm": 299.9615478515625,
      "learning_rate": 4.638291690877398e-06,
      "logits/chosen": -0.857986569404602,
      "logits/rejected": -0.9681297540664673,
      "logps/chosen": -73.25657653808594,
      "logps/rejected": -80.2601547241211,
      "loss": 13.4933,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.00817027036100626,
      "rewards/margins": 0.04146546870470047,
      "rewards/rejected": -0.04963573440909386,
      "step": 249
    },
    {
      "epoch": 0.14520532032293662,
      "grad_norm": 302.1356201171875,
      "learning_rate": 4.6368390470656596e-06,
      "logits/chosen": -0.9270285367965698,
      "logits/rejected": -1.053038239479065,
      "logps/chosen": -65.88447570800781,
      "logps/rejected": -72.62715148925781,
      "loss": 14.0395,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.032028716057538986,
      "rewards/margins": -0.014700132422149181,
      "rewards/rejected": -0.01732858456671238,
      "step": 250
    },
    {
      "epoch": 0.14578614160422837,
      "grad_norm": 296.97564697265625,
      "learning_rate": 4.635386403253922e-06,
      "logits/chosen": -0.8831275105476379,
      "logits/rejected": -0.8724244832992554,
      "logps/chosen": -66.05812072753906,
      "logps/rejected": -67.20362854003906,
      "loss": 13.9448,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0004748806240968406,
      "rewards/margins": -0.0028558894991874695,
      "rewards/rejected": 0.0033307753037661314,
      "step": 251
    },
    {
      "epoch": 0.14636696288552012,
      "grad_norm": 315.11810302734375,
      "learning_rate": 4.633933759442185e-06,
      "logits/chosen": -1.0385617017745972,
      "logits/rejected": -1.0641318559646606,
      "logps/chosen": -72.60139465332031,
      "logps/rejected": -70.97787475585938,
      "loss": 13.838,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.012739667668938637,
      "rewards/margins": 0.007387256715446711,
      "rewards/rejected": -0.02012692391872406,
      "step": 252
    },
    {
      "epoch": 0.14694778416681187,
      "grad_norm": 315.47259521484375,
      "learning_rate": 4.632481115630448e-06,
      "logits/chosen": -0.8671859502792358,
      "logits/rejected": -0.8790918588638306,
      "logps/chosen": -80.53587341308594,
      "logps/rejected": -78.8593978881836,
      "loss": 13.9845,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.034586913883686066,
      "rewards/margins": -0.007644425146281719,
      "rewards/rejected": -0.02694249153137207,
      "step": 253
    },
    {
      "epoch": 0.14752860544810362,
      "grad_norm": 310.9654541015625,
      "learning_rate": 4.6310284718187105e-06,
      "logits/chosen": -0.8506280183792114,
      "logits/rejected": -0.9469968676567078,
      "logps/chosen": -83.29402923583984,
      "logps/rejected": -84.33534240722656,
      "loss": 13.4946,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.028773630037903786,
      "rewards/margins": 0.0417972095310688,
      "rewards/rejected": -0.07057084143161774,
      "step": 254
    },
    {
      "epoch": 0.14810942672939537,
      "grad_norm": 304.36968994140625,
      "learning_rate": 4.629575828006973e-06,
      "logits/chosen": -0.7221881151199341,
      "logits/rejected": -0.8106364011764526,
      "logps/chosen": -73.52366638183594,
      "logps/rejected": -76.93601989746094,
      "loss": 13.6697,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.017531022429466248,
      "rewards/margins": 0.022295940667390823,
      "rewards/rejected": -0.03982696682214737,
      "step": 255
    },
    {
      "epoch": 0.14869024801068711,
      "grad_norm": 731.6671752929688,
      "learning_rate": 4.628123184195236e-06,
      "logits/chosen": -1.0281916856765747,
      "logits/rejected": -1.0932250022888184,
      "logps/chosen": -73.1572036743164,
      "logps/rejected": -76.2696304321289,
      "loss": 13.9093,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009217451326549053,
      "rewards/margins": -0.0013483152724802494,
      "rewards/rejected": -0.007869137451052666,
      "step": 256
    },
    {
      "epoch": 0.14927106929197886,
      "grad_norm": 317.27630615234375,
      "learning_rate": 4.626670540383498e-06,
      "logits/chosen": -0.7391899824142456,
      "logits/rejected": -0.8331745862960815,
      "logps/chosen": -70.43421936035156,
      "logps/rejected": -76.86418151855469,
      "loss": 13.9393,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.015171433798968792,
      "rewards/margins": -0.0031314187217503786,
      "rewards/rejected": 0.018302852287888527,
      "step": 257
    },
    {
      "epoch": 0.1498518905732706,
      "grad_norm": 323.65386962890625,
      "learning_rate": 4.625217896571761e-06,
      "logits/chosen": -0.8975432515144348,
      "logits/rejected": -1.0589958429336548,
      "logps/chosen": -75.39137268066406,
      "logps/rejected": -71.28562927246094,
      "loss": 13.5174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.009832754731178284,
      "rewards/margins": 0.03878428786993027,
      "rewards/rejected": -0.04861704260110855,
      "step": 258
    },
    {
      "epoch": 0.15043271185456236,
      "grad_norm": 304.92144775390625,
      "learning_rate": 4.6237652527600234e-06,
      "logits/chosen": -0.7835612297058105,
      "logits/rejected": -0.8767670392990112,
      "logps/chosen": -73.34691619873047,
      "logps/rejected": -70.6404037475586,
      "loss": 14.2398,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04404080659151077,
      "rewards/margins": -0.032345980405807495,
      "rewards/rejected": -0.011694823391735554,
      "step": 259
    },
    {
      "epoch": 0.1510135331358541,
      "grad_norm": 324.8614196777344,
      "learning_rate": 4.622312608948286e-06,
      "logits/chosen": -0.6987928152084351,
      "logits/rejected": -0.765534520149231,
      "logps/chosen": -72.92799377441406,
      "logps/rejected": -69.84391784667969,
      "loss": 13.887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.027519341558218002,
      "rewards/margins": 0.0016624340787529945,
      "rewards/rejected": -0.02918177843093872,
      "step": 260
    },
    {
      "epoch": 0.15159435441714583,
      "grad_norm": 311.1237487792969,
      "learning_rate": 4.620859965136549e-06,
      "logits/chosen": -0.7852567434310913,
      "logits/rejected": -0.9455236196517944,
      "logps/chosen": -72.65814971923828,
      "logps/rejected": -79.49076080322266,
      "loss": 13.4405,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02082516811788082,
      "rewards/margins": 0.046132300049066544,
      "rewards/rejected": -0.06695746630430222,
      "step": 261
    },
    {
      "epoch": 0.15217517569843758,
      "grad_norm": 312.7984924316406,
      "learning_rate": 4.619407321324812e-06,
      "logits/chosen": -0.9117915034294128,
      "logits/rejected": -0.9733545184135437,
      "logps/chosen": -77.04357147216797,
      "logps/rejected": -66.96760559082031,
      "loss": 14.3348,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0389360710978508,
      "rewards/margins": -0.043142665177583694,
      "rewards/rejected": 0.0042065903544425964,
      "step": 262
    },
    {
      "epoch": 0.15275599697972933,
      "grad_norm": 338.8244934082031,
      "learning_rate": 4.6179546775130744e-06,
      "logits/chosen": -0.8544296026229858,
      "logits/rejected": -0.8088324666023254,
      "logps/chosen": -76.08251190185547,
      "logps/rejected": -74.12071990966797,
      "loss": 13.896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017584245651960373,
      "rewards/margins": -0.000663819897454232,
      "rewards/rejected": -0.0169204268604517,
      "step": 263
    },
    {
      "epoch": 0.15333681826102108,
      "grad_norm": 286.11407470703125,
      "learning_rate": 4.616502033701336e-06,
      "logits/chosen": -0.8225248456001282,
      "logits/rejected": -0.8068425059318542,
      "logps/chosen": -75.96072387695312,
      "logps/rejected": -70.82938385009766,
      "loss": 13.5013,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02497541345655918,
      "rewards/margins": 0.04040759429335594,
      "rewards/rejected": -0.01543218083679676,
      "step": 264
    },
    {
      "epoch": 0.15391763954231283,
      "grad_norm": 319.89794921875,
      "learning_rate": 4.615049389889599e-06,
      "logits/chosen": -0.9208014607429504,
      "logits/rejected": -0.9296444654464722,
      "logps/chosen": -75.55086517333984,
      "logps/rejected": -79.83537292480469,
      "loss": 13.6912,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0359053872525692,
      "rewards/margins": 0.024030333384871483,
      "rewards/rejected": -0.05993572622537613,
      "step": 265
    },
    {
      "epoch": 0.15449846082360458,
      "grad_norm": 360.5916442871094,
      "learning_rate": 4.613596746077862e-06,
      "logits/chosen": -0.745102047920227,
      "logits/rejected": -0.7995504140853882,
      "logps/chosen": -75.1971206665039,
      "logps/rejected": -68.94092559814453,
      "loss": 13.8718,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02639504335820675,
      "rewards/margins": 0.0018470294307917356,
      "rewards/rejected": -0.028242075815796852,
      "step": 266
    },
    {
      "epoch": 0.15507928210489633,
      "grad_norm": 306.31646728515625,
      "learning_rate": 4.612144102266125e-06,
      "logits/chosen": -0.7601212859153748,
      "logits/rejected": -0.8015406727790833,
      "logps/chosen": -79.32325744628906,
      "logps/rejected": -73.96139526367188,
      "loss": 14.0699,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.05416913703083992,
      "rewards/margins": -0.017042648047208786,
      "rewards/rejected": -0.037126488983631134,
      "step": 267
    },
    {
      "epoch": 0.15566010338618808,
      "grad_norm": 371.703125,
      "learning_rate": 4.610691458454387e-06,
      "logits/chosen": -0.9163106679916382,
      "logits/rejected": -1.0520881414413452,
      "logps/chosen": -78.73876953125,
      "logps/rejected": -79.40687561035156,
      "loss": 13.5618,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009102868847548962,
      "rewards/margins": 0.032982636243104935,
      "rewards/rejected": -0.04208550602197647,
      "step": 268
    },
    {
      "epoch": 0.15624092466747982,
      "grad_norm": 309.1504211425781,
      "learning_rate": 4.60923881464265e-06,
      "logits/chosen": -0.9318111538887024,
      "logits/rejected": -0.9711005091667175,
      "logps/chosen": -74.52928924560547,
      "logps/rejected": -78.32316589355469,
      "loss": 13.5172,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03735839203000069,
      "rewards/margins": 0.03915861248970032,
      "rewards/rejected": -0.0765170007944107,
      "step": 269
    },
    {
      "epoch": 0.15682174594877157,
      "grad_norm": 297.4718017578125,
      "learning_rate": 4.607786170830912e-06,
      "logits/chosen": -0.9283093214035034,
      "logits/rejected": -0.9299993515014648,
      "logps/chosen": -73.22312927246094,
      "logps/rejected": -69.17174530029297,
      "loss": 13.9256,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0193580724298954,
      "rewards/margins": -0.0019154436886310577,
      "rewards/rejected": -0.017442626878619194,
      "step": 270
    },
    {
      "epoch": 0.15740256723006332,
      "grad_norm": 315.2553405761719,
      "learning_rate": 4.606333527019175e-06,
      "logits/chosen": -0.8597780466079712,
      "logits/rejected": -0.811455249786377,
      "logps/chosen": -67.54701232910156,
      "logps/rejected": -77.47270965576172,
      "loss": 13.5603,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0011511326301842928,
      "rewards/margins": 0.03523620590567589,
      "rewards/rejected": -0.03638733923435211,
      "step": 271
    },
    {
      "epoch": 0.15798338851135504,
      "grad_norm": 788.3867797851562,
      "learning_rate": 4.6048808832074375e-06,
      "logits/chosen": -0.8214397430419922,
      "logits/rejected": -0.791312575340271,
      "logps/chosen": -69.90260314941406,
      "logps/rejected": -70.34420776367188,
      "loss": 13.6075,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.010176298208534718,
      "rewards/margins": 0.028864461928606033,
      "rewards/rejected": -0.03904075548052788,
      "step": 272
    },
    {
      "epoch": 0.1585642097926468,
      "grad_norm": 301.57794189453125,
      "learning_rate": 4.603428239395701e-06,
      "logits/chosen": -0.8692893981933594,
      "logits/rejected": -0.8743621706962585,
      "logps/chosen": -68.12577056884766,
      "logps/rejected": -66.91731262207031,
      "loss": 13.7415,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.013723989017307758,
      "rewards/margins": 0.015195539221167564,
      "rewards/rejected": -0.028919529169797897,
      "step": 273
    },
    {
      "epoch": 0.15914503107393854,
      "grad_norm": 290.8895263671875,
      "learning_rate": 4.601975595583964e-06,
      "logits/chosen": -0.8542564511299133,
      "logits/rejected": -0.8347529172897339,
      "logps/chosen": -73.254150390625,
      "logps/rejected": -68.77384948730469,
      "loss": 13.5118,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0077955336309969425,
      "rewards/margins": 0.03801111876964569,
      "rewards/rejected": -0.030215587466955185,
      "step": 274
    },
    {
      "epoch": 0.1597258523552303,
      "grad_norm": 299.75982666015625,
      "learning_rate": 4.600522951772226e-06,
      "logits/chosen": -0.9737634658813477,
      "logits/rejected": -1.0128427743911743,
      "logps/chosen": -72.66883850097656,
      "logps/rejected": -72.12713623046875,
      "loss": 14.1505,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04277346283197403,
      "rewards/margins": -0.02521767094731331,
      "rewards/rejected": -0.01755579002201557,
      "step": 275
    },
    {
      "epoch": 0.16030667363652204,
      "grad_norm": 317.3355712890625,
      "learning_rate": 4.5990703079604885e-06,
      "logits/chosen": -0.7819968461990356,
      "logits/rejected": -0.7388423681259155,
      "logps/chosen": -68.65778350830078,
      "logps/rejected": -69.62420654296875,
      "loss": 14.0828,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.014692326076328754,
      "rewards/margins": -0.017574016004800797,
      "rewards/rejected": 0.03226633742451668,
      "step": 276
    },
    {
      "epoch": 0.1608874949178138,
      "grad_norm": 300.1111145019531,
      "learning_rate": 4.597617664148751e-06,
      "logits/chosen": -0.8221060037612915,
      "logits/rejected": -0.8107389211654663,
      "logps/chosen": -64.23518371582031,
      "logps/rejected": -74.13258361816406,
      "loss": 13.5993,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01461729221045971,
      "rewards/margins": 0.028885364532470703,
      "rewards/rejected": -0.014268075115978718,
      "step": 277
    },
    {
      "epoch": 0.16146831619910554,
      "grad_norm": 324.15899658203125,
      "learning_rate": 4.596165020337014e-06,
      "logits/chosen": -0.8910449147224426,
      "logits/rejected": -1.0317879915237427,
      "logps/chosen": -72.35569763183594,
      "logps/rejected": -86.64521789550781,
      "loss": 13.3693,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04237177222967148,
      "rewards/margins": 0.05554167553782463,
      "rewards/rejected": -0.09791344404220581,
      "step": 278
    },
    {
      "epoch": 0.1620491374803973,
      "grad_norm": 335.16619873046875,
      "learning_rate": 4.594712376525277e-06,
      "logits/chosen": -0.8945894241333008,
      "logits/rejected": -0.9022412300109863,
      "logps/chosen": -72.68575286865234,
      "logps/rejected": -77.0567626953125,
      "loss": 13.5153,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0022069262340664864,
      "rewards/margins": 0.03883010149002075,
      "rewards/rejected": -0.03662317246198654,
      "step": 279
    },
    {
      "epoch": 0.16262995876168904,
      "grad_norm": 296.8701477050781,
      "learning_rate": 4.5932597327135395e-06,
      "logits/chosen": -0.8403556942939758,
      "logits/rejected": -0.8022698163986206,
      "logps/chosen": -68.22055053710938,
      "logps/rejected": -68.78074645996094,
      "loss": 13.5981,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01986340805888176,
      "rewards/margins": 0.03163480758666992,
      "rewards/rejected": -0.05149821564555168,
      "step": 280
    },
    {
      "epoch": 0.16321078004298079,
      "grad_norm": 305.6988830566406,
      "learning_rate": 4.591807088901802e-06,
      "logits/chosen": -0.8042858839035034,
      "logits/rejected": -0.7476822137832642,
      "logps/chosen": -64.76155090332031,
      "logps/rejected": -72.77494812011719,
      "loss": 13.7642,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.013462590985000134,
      "rewards/margins": 0.014226732775568962,
      "rewards/rejected": -0.02768932655453682,
      "step": 281
    },
    {
      "epoch": 0.16379160132427253,
      "grad_norm": 317.730712890625,
      "learning_rate": 4.590354445090064e-06,
      "logits/chosen": -0.8070831298828125,
      "logits/rejected": -0.7661058306694031,
      "logps/chosen": -69.77413940429688,
      "logps/rejected": -69.04270935058594,
      "loss": 13.623,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.008160887286067009,
      "rewards/margins": 0.029747169464826584,
      "rewards/rejected": -0.021586284041404724,
      "step": 282
    },
    {
      "epoch": 0.16437242260556426,
      "grad_norm": 313.9194641113281,
      "learning_rate": 4.588901801278327e-06,
      "logits/chosen": -0.8869432210922241,
      "logits/rejected": -1.0530153512954712,
      "logps/chosen": -76.52225494384766,
      "logps/rejected": -70.056884765625,
      "loss": 13.6882,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.044385336339473724,
      "rewards/margins": 0.02604733407497406,
      "rewards/rejected": -0.07043267786502838,
      "step": 283
    },
    {
      "epoch": 0.164953243886856,
      "grad_norm": 326.5664978027344,
      "learning_rate": 4.58744915746659e-06,
      "logits/chosen": -0.7581676244735718,
      "logits/rejected": -0.7644235491752625,
      "logps/chosen": -75.07527160644531,
      "logps/rejected": -67.26031494140625,
      "loss": 14.2293,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.024589911103248596,
      "rewards/margins": -0.03345941752195358,
      "rewards/rejected": 0.008869504556059837,
      "step": 284
    },
    {
      "epoch": 0.16553406516814775,
      "grad_norm": 331.2121276855469,
      "learning_rate": 4.585996513654852e-06,
      "logits/chosen": -0.6432844400405884,
      "logits/rejected": -0.6698486804962158,
      "logps/chosen": -79.6335220336914,
      "logps/rejected": -84.1219253540039,
      "loss": 13.6497,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.007236090488731861,
      "rewards/margins": 0.02394806034862995,
      "rewards/rejected": -0.03118414804339409,
      "step": 285
    },
    {
      "epoch": 0.1661148864494395,
      "grad_norm": 325.1278381347656,
      "learning_rate": 4.584543869843115e-06,
      "logits/chosen": -0.805732250213623,
      "logits/rejected": -0.810941219329834,
      "logps/chosen": -81.93310546875,
      "logps/rejected": -72.44585418701172,
      "loss": 14.3678,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.03649924322962761,
      "rewards/margins": -0.046101175248622894,
      "rewards/rejected": 0.009601928293704987,
      "step": 286
    },
    {
      "epoch": 0.16669570773073125,
      "grad_norm": 304.5572814941406,
      "learning_rate": 4.583091226031378e-06,
      "logits/chosen": -0.7995644211769104,
      "logits/rejected": -0.8172422647476196,
      "logps/chosen": -72.41849517822266,
      "logps/rejected": -67.6212387084961,
      "loss": 13.8286,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.016089126467704773,
      "rewards/margins": 0.00835293810814619,
      "rewards/rejected": 0.007736186496913433,
      "step": 287
    },
    {
      "epoch": 0.167276529012023,
      "grad_norm": 338.3421936035156,
      "learning_rate": 4.581638582219641e-06,
      "logits/chosen": -0.7788872122764587,
      "logits/rejected": -0.8800986409187317,
      "logps/chosen": -77.22239685058594,
      "logps/rejected": -84.37377166748047,
      "loss": 13.6912,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01143337320536375,
      "rewards/margins": 0.02296261489391327,
      "rewards/rejected": -0.034395989030599594,
      "step": 288
    },
    {
      "epoch": 0.16785735029331475,
      "grad_norm": 303.7924499511719,
      "learning_rate": 4.5801859384079025e-06,
      "logits/chosen": -0.8267248868942261,
      "logits/rejected": -0.8682713508605957,
      "logps/chosen": -76.82869720458984,
      "logps/rejected": -75.02543640136719,
      "loss": 13.9469,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.009327888488769531,
      "rewards/margins": -0.004273596219718456,
      "rewards/rejected": 0.013601483777165413,
      "step": 289
    },
    {
      "epoch": 0.1684381715746065,
      "grad_norm": 310.5744323730469,
      "learning_rate": 4.578733294596165e-06,
      "logits/chosen": -0.8373514413833618,
      "logits/rejected": -0.8355886340141296,
      "logps/chosen": -78.4135971069336,
      "logps/rejected": -68.44206237792969,
      "loss": 13.3873,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.01929556205868721,
      "rewards/margins": 0.05108966678380966,
      "rewards/rejected": -0.07038523256778717,
      "step": 290
    },
    {
      "epoch": 0.16901899285589825,
      "grad_norm": 294.162353515625,
      "learning_rate": 4.577280650784428e-06,
      "logits/chosen": -0.8757956624031067,
      "logits/rejected": -0.9635500907897949,
      "logps/chosen": -79.70155334472656,
      "logps/rejected": -72.04487609863281,
      "loss": 13.3423,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.015520468354225159,
      "rewards/margins": 0.056552886962890625,
      "rewards/rejected": -0.041032422333955765,
      "step": 291
    },
    {
      "epoch": 0.16959981413719,
      "grad_norm": 406.17034912109375,
      "learning_rate": 4.575828006972691e-06,
      "logits/chosen": -0.712080717086792,
      "logits/rejected": -0.6416457891464233,
      "logps/chosen": -74.07041931152344,
      "logps/rejected": -75.76925659179688,
      "loss": 13.8944,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.009814701043069363,
      "rewards/margins": 0.0012444716412574053,
      "rewards/rejected": -0.0110591696575284,
      "step": 292
    },
    {
      "epoch": 0.17018063541848175,
      "grad_norm": 319.25677490234375,
      "learning_rate": 4.5743753631609535e-06,
      "logits/chosen": -0.978374183177948,
      "logits/rejected": -0.9963283538818359,
      "logps/chosen": -68.7942886352539,
      "logps/rejected": -71.56004333496094,
      "loss": 13.9775,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.004177446011453867,
      "rewards/margins": -0.006889923010021448,
      "rewards/rejected": 0.002712479094043374,
      "step": 293
    },
    {
      "epoch": 0.17076145669977347,
      "grad_norm": 333.2608642578125,
      "learning_rate": 4.572922719349216e-06,
      "logits/chosen": -0.9167648553848267,
      "logits/rejected": -0.8717905282974243,
      "logps/chosen": -70.68975067138672,
      "logps/rejected": -77.90010070800781,
      "loss": 13.5974,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.002308635041117668,
      "rewards/margins": 0.027850687503814697,
      "rewards/rejected": -0.02554205060005188,
      "step": 294
    },
    {
      "epoch": 0.17134227798106522,
      "grad_norm": 312.9326477050781,
      "learning_rate": 4.571470075537478e-06,
      "logits/chosen": -0.7617141008377075,
      "logits/rejected": -0.8908483386039734,
      "logps/chosen": -70.78837585449219,
      "logps/rejected": -72.64606475830078,
      "loss": 14.292,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.008612537756562233,
      "rewards/margins": -0.03856853395700455,
      "rewards/rejected": 0.029956001788377762,
      "step": 295
    },
    {
      "epoch": 0.17192309926235697,
      "grad_norm": 308.8948669433594,
      "learning_rate": 4.570017431725741e-06,
      "logits/chosen": -0.913497269153595,
      "logits/rejected": -0.8606641888618469,
      "logps/chosen": -70.07915496826172,
      "logps/rejected": -82.69215393066406,
      "loss": 13.7394,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.011818322353065014,
      "rewards/margins": 0.017899369820952415,
      "rewards/rejected": -0.029717693105340004,
      "step": 296
    },
    {
      "epoch": 0.17250392054364871,
      "grad_norm": 291.6385803222656,
      "learning_rate": 4.568564787914004e-06,
      "logits/chosen": -0.8296724557876587,
      "logits/rejected": -0.9198773503303528,
      "logps/chosen": -67.22807312011719,
      "logps/rejected": -78.60759735107422,
      "loss": 13.3225,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0034950252156704664,
      "rewards/margins": 0.06311126798391342,
      "rewards/rejected": -0.0666062980890274,
      "step": 297
    },
    {
      "epoch": 0.17308474182494046,
      "grad_norm": 291.3044738769531,
      "learning_rate": 4.567112144102266e-06,
      "logits/chosen": -0.8050098419189453,
      "logits/rejected": -0.7676219940185547,
      "logps/chosen": -67.16458129882812,
      "logps/rejected": -78.90528869628906,
      "loss": 13.2096,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02983633242547512,
      "rewards/margins": 0.07233790308237076,
      "rewards/rejected": -0.042501576244831085,
      "step": 298
    },
    {
      "epoch": 0.1736655631062322,
      "grad_norm": 314.0892333984375,
      "learning_rate": 4.565659500290529e-06,
      "logits/chosen": -1.0540732145309448,
      "logits/rejected": -0.9160014986991882,
      "logps/chosen": -71.54045104980469,
      "logps/rejected": -75.05534362792969,
      "loss": 13.7491,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009140492416918278,
      "rewards/margins": 0.015866661444306374,
      "rewards/rejected": -0.006726170424371958,
      "step": 299
    },
    {
      "epoch": 0.17424638438752396,
      "grad_norm": 310.53985595703125,
      "learning_rate": 4.564206856478792e-06,
      "logits/chosen": -0.8692498207092285,
      "logits/rejected": -0.8690904378890991,
      "logps/chosen": -71.8248062133789,
      "logps/rejected": -82.47134399414062,
      "loss": 12.8867,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.015918642282485962,
      "rewards/margins": 0.10448668152093887,
      "rewards/rejected": -0.08856804668903351,
      "step": 300
    },
    {
      "epoch": 0.1748272056688157,
      "grad_norm": 298.11541748046875,
      "learning_rate": 4.562754212667055e-06,
      "logits/chosen": -0.8415626287460327,
      "logits/rejected": -0.8000283241271973,
      "logps/chosen": -69.10453033447266,
      "logps/rejected": -65.39994812011719,
      "loss": 13.946,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.024037543684244156,
      "rewards/margins": -0.00543618481606245,
      "rewards/rejected": -0.018601354211568832,
      "step": 301
    },
    {
      "epoch": 0.17540802695010746,
      "grad_norm": 286.9059753417969,
      "learning_rate": 4.5613015688553165e-06,
      "logits/chosen": -0.8321945071220398,
      "logits/rejected": -0.8791966438293457,
      "logps/chosen": -66.0924301147461,
      "logps/rejected": -70.165283203125,
      "loss": 13.9943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0016370766097679734,
      "rewards/margins": -0.008972769603133202,
      "rewards/rejected": 0.007335691247135401,
      "step": 302
    },
    {
      "epoch": 0.1759888482313992,
      "grad_norm": 324.43927001953125,
      "learning_rate": 4.559848925043579e-06,
      "logits/chosen": -0.8788312077522278,
      "logits/rejected": -0.8677918314933777,
      "logps/chosen": -70.78472137451172,
      "logps/rejected": -70.58416748046875,
      "loss": 13.756,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.015381020493805408,
      "rewards/margins": 0.013850994408130646,
      "rewards/rejected": -0.02923201583325863,
      "step": 303
    },
    {
      "epoch": 0.17656966951269096,
      "grad_norm": 333.6717224121094,
      "learning_rate": 4.558396281231842e-06,
      "logits/chosen": -0.9329641461372375,
      "logits/rejected": -0.8686521649360657,
      "logps/chosen": -73.89896392822266,
      "logps/rejected": -78.4896011352539,
      "loss": 13.8188,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.00456998823210597,
      "rewards/margins": 0.008396768942475319,
      "rewards/rejected": -0.0038267814088612795,
      "step": 304
    },
    {
      "epoch": 0.17715049079398268,
      "grad_norm": 306.7808532714844,
      "learning_rate": 4.556943637420105e-06,
      "logits/chosen": -0.8817129135131836,
      "logits/rejected": -0.7859727144241333,
      "logps/chosen": -70.53137969970703,
      "logps/rejected": -73.36188507080078,
      "loss": 13.6795,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0007974056643433869,
      "rewards/margins": 0.025630056858062744,
      "rewards/rejected": -0.024832649156451225,
      "step": 305
    },
    {
      "epoch": 0.17773131207527443,
      "grad_norm": 299.79339599609375,
      "learning_rate": 4.5554909936083675e-06,
      "logits/chosen": -0.8397809863090515,
      "logits/rejected": -0.962628960609436,
      "logps/chosen": -74.5028305053711,
      "logps/rejected": -72.23627471923828,
      "loss": 14.1074,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.013882589526474476,
      "rewards/margins": -0.022180356085300446,
      "rewards/rejected": 0.03606294468045235,
      "step": 306
    },
    {
      "epoch": 0.17831213335656618,
      "grad_norm": 305.6503601074219,
      "learning_rate": 4.55403834979663e-06,
      "logits/chosen": -0.9371621012687683,
      "logits/rejected": -0.8676670789718628,
      "logps/chosen": -68.56401824951172,
      "logps/rejected": -66.05886840820312,
      "loss": 13.7674,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.023868542164564133,
      "rewards/margins": 0.013151749968528748,
      "rewards/rejected": 0.010716790333390236,
      "step": 307
    },
    {
      "epoch": 0.17889295463785793,
      "grad_norm": 316.7124328613281,
      "learning_rate": 4.552585705984893e-06,
      "logits/chosen": -0.795932412147522,
      "logits/rejected": -0.8041373491287231,
      "logps/chosen": -75.10924530029297,
      "logps/rejected": -74.57756042480469,
      "loss": 14.0814,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.026643935590982437,
      "rewards/margins": -0.018288511782884598,
      "rewards/rejected": -0.008355428464710712,
      "step": 308
    },
    {
      "epoch": 0.17947377591914968,
      "grad_norm": 316.54193115234375,
      "learning_rate": 4.551133062173155e-06,
      "logits/chosen": -0.7379294037818909,
      "logits/rejected": -0.8033340573310852,
      "logps/chosen": -70.75437927246094,
      "logps/rejected": -68.13688659667969,
      "loss": 14.334,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.04628816246986389,
      "rewards/margins": -0.04288000240921974,
      "rewards/rejected": -0.0034081649500876665,
      "step": 309
    },
    {
      "epoch": 0.18005459720044142,
      "grad_norm": 320.3726501464844,
      "learning_rate": 4.549680418361418e-06,
      "logits/chosen": -0.8026307225227356,
      "logits/rejected": -0.7256749272346497,
      "logps/chosen": -77.1530532836914,
      "logps/rejected": -71.47651672363281,
      "loss": 14.3986,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.03423710912466049,
      "rewards/margins": -0.04747765138745308,
      "rewards/rejected": 0.013240538537502289,
      "step": 310
    },
    {
      "epoch": 0.18063541848173317,
      "grad_norm": 333.2999572753906,
      "learning_rate": 4.5482277745496804e-06,
      "logits/chosen": -0.9229456186294556,
      "logits/rejected": -0.9455530047416687,
      "logps/chosen": -80.14155578613281,
      "logps/rejected": -68.71078491210938,
      "loss": 14.274,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03716535493731499,
      "rewards/margins": -0.035925768315792084,
      "rewards/rejected": -0.001239586272276938,
      "step": 311
    },
    {
      "epoch": 0.18121623976302492,
      "grad_norm": 344.1839904785156,
      "learning_rate": 4.546775130737943e-06,
      "logits/chosen": -0.7100509405136108,
      "logits/rejected": -0.7682685852050781,
      "logps/chosen": -70.23936462402344,
      "logps/rejected": -70.01929473876953,
      "loss": 13.9029,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0003937946748919785,
      "rewards/margins": -7.176501821959391e-05,
      "rewards/rejected": -0.0003220273065380752,
      "step": 312
    },
    {
      "epoch": 0.18179706104431667,
      "grad_norm": 311.8206787109375,
      "learning_rate": 4.545322486926206e-06,
      "logits/chosen": -0.786939263343811,
      "logits/rejected": -0.7103012800216675,
      "logps/chosen": -73.2970962524414,
      "logps/rejected": -68.45187377929688,
      "loss": 14.034,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01686491258442402,
      "rewards/margins": -0.013414248824119568,
      "rewards/rejected": -0.0034506588708609343,
      "step": 313
    },
    {
      "epoch": 0.18237788232560842,
      "grad_norm": 290.2313537597656,
      "learning_rate": 4.543869843114469e-06,
      "logits/chosen": -0.8251843452453613,
      "logits/rejected": -0.8637291193008423,
      "logps/chosen": -76.97846984863281,
      "logps/rejected": -66.21952819824219,
      "loss": 13.9921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003126763505861163,
      "rewards/margins": -0.010613595135509968,
      "rewards/rejected": 0.007486830465495586,
      "step": 314
    },
    {
      "epoch": 0.18295870360690017,
      "grad_norm": 357.3050842285156,
      "learning_rate": 4.542417199302731e-06,
      "logits/chosen": -0.7790865898132324,
      "logits/rejected": -0.8820877075195312,
      "logps/chosen": -69.9126968383789,
      "logps/rejected": -73.85781860351562,
      "loss": 13.243,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.004135083872824907,
      "rewards/margins": 0.07195592671632767,
      "rewards/rejected": -0.06782083958387375,
      "step": 315
    },
    {
      "epoch": 0.1835395248881919,
      "grad_norm": 304.0280456542969,
      "learning_rate": 4.540964555490994e-06,
      "logits/chosen": -0.8775313496589661,
      "logits/rejected": -0.966948390007019,
      "logps/chosen": -73.7503890991211,
      "logps/rejected": -71.78192138671875,
      "loss": 13.2604,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02027755416929722,
      "rewards/margins": 0.07092587649822235,
      "rewards/rejected": -0.050648320466279984,
      "step": 316
    },
    {
      "epoch": 0.18412034616948364,
      "grad_norm": 307.97015380859375,
      "learning_rate": 4.539511911679257e-06,
      "logits/chosen": -0.8565672636032104,
      "logits/rejected": -0.9156128764152527,
      "logps/chosen": -73.54744720458984,
      "logps/rejected": -73.94361877441406,
      "loss": 13.7971,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005255700554698706,
      "rewards/margins": 0.008574297651648521,
      "rewards/rejected": -0.013830000534653664,
      "step": 317
    },
    {
      "epoch": 0.1847011674507754,
      "grad_norm": 312.7808837890625,
      "learning_rate": 4.53805926786752e-06,
      "logits/chosen": -0.8373235464096069,
      "logits/rejected": -0.8614371418952942,
      "logps/chosen": -71.75985717773438,
      "logps/rejected": -80.21531677246094,
      "loss": 13.8805,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002429866697639227,
      "rewards/margins": 0.001796521246433258,
      "rewards/rejected": 0.0006333448109216988,
      "step": 318
    },
    {
      "epoch": 0.18528198873206714,
      "grad_norm": 322.60736083984375,
      "learning_rate": 4.536606624055782e-06,
      "logits/chosen": -0.8135954141616821,
      "logits/rejected": -0.8624083399772644,
      "logps/chosen": -67.81532287597656,
      "logps/rejected": -81.07484436035156,
      "loss": 13.4658,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.008448420092463493,
      "rewards/margins": 0.04661915823817253,
      "rewards/rejected": -0.055067580193281174,
      "step": 319
    },
    {
      "epoch": 0.1858628100133589,
      "grad_norm": 289.1242370605469,
      "learning_rate": 4.535153980244045e-06,
      "logits/chosen": -0.7294256687164307,
      "logits/rejected": -0.7239043116569519,
      "logps/chosen": -70.69343566894531,
      "logps/rejected": -76.38871765136719,
      "loss": 13.2891,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04026845842599869,
      "rewards/margins": 0.06055659055709839,
      "rewards/rejected": -0.020288124680519104,
      "step": 320
    },
    {
      "epoch": 0.18644363129465064,
      "grad_norm": 316.1600341796875,
      "learning_rate": 4.533701336432307e-06,
      "logits/chosen": -0.7641903162002563,
      "logits/rejected": -0.8571245074272156,
      "logps/chosen": -72.52635192871094,
      "logps/rejected": -72.56694030761719,
      "loss": 13.0747,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.040480513125658035,
      "rewards/margins": 0.08481685817241669,
      "rewards/rejected": -0.04433634877204895,
      "step": 321
    },
    {
      "epoch": 0.18702445257594238,
      "grad_norm": 331.61419677734375,
      "learning_rate": 4.53224869262057e-06,
      "logits/chosen": -0.7687914967536926,
      "logits/rejected": -0.862636387348175,
      "logps/chosen": -69.64253997802734,
      "logps/rejected": -72.28370666503906,
      "loss": 13.9979,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.030815565958619118,
      "rewards/margins": -0.009632897563278675,
      "rewards/rejected": -0.021182667464017868,
      "step": 322
    },
    {
      "epoch": 0.18760527385723413,
      "grad_norm": 294.9023742675781,
      "learning_rate": 4.5307960488088326e-06,
      "logits/chosen": -0.7634553909301758,
      "logits/rejected": -0.8270981907844543,
      "logps/chosen": -72.29499816894531,
      "logps/rejected": -74.75524139404297,
      "loss": 13.8647,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.014828644692897797,
      "rewards/margins": 0.004178445786237717,
      "rewards/rejected": -0.019007090479135513,
      "step": 323
    },
    {
      "epoch": 0.18818609513852588,
      "grad_norm": 303.8163757324219,
      "learning_rate": 4.529343404997095e-06,
      "logits/chosen": -0.7258373498916626,
      "logits/rejected": -0.7167456746101379,
      "logps/chosen": -71.8295669555664,
      "logps/rejected": -68.2274169921875,
      "loss": 14.0144,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04376727342605591,
      "rewards/margins": -0.011802466586232185,
      "rewards/rejected": -0.03196480870246887,
      "step": 324
    },
    {
      "epoch": 0.18876691641981763,
      "grad_norm": 304.3807678222656,
      "learning_rate": 4.527890761185358e-06,
      "logits/chosen": -0.8415002822875977,
      "logits/rejected": -0.8635972738265991,
      "logps/chosen": -76.11531829833984,
      "logps/rejected": -74.70405578613281,
      "loss": 14.0264,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02958356961607933,
      "rewards/margins": -0.012393072247505188,
      "rewards/rejected": -0.017190497368574142,
      "step": 325
    },
    {
      "epoch": 0.18934773770110938,
      "grad_norm": 315.16876220703125,
      "learning_rate": 4.526438117373621e-06,
      "logits/chosen": -0.7613179683685303,
      "logits/rejected": -0.776989221572876,
      "logps/chosen": -67.979248046875,
      "logps/rejected": -69.2055435180664,
      "loss": 13.738,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007391949184238911,
      "rewards/margins": 0.015842467546463013,
      "rewards/rejected": -0.008450517430901527,
      "step": 326
    },
    {
      "epoch": 0.1899285589824011,
      "grad_norm": 310.1661682128906,
      "learning_rate": 4.524985473561883e-06,
      "logits/chosen": -0.9094734191894531,
      "logits/rejected": -0.8339530825614929,
      "logps/chosen": -76.08878326416016,
      "logps/rejected": -70.17434692382812,
      "loss": 13.821,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.020542889833450317,
      "rewards/margins": 0.006295947823673487,
      "rewards/rejected": 0.014246943406760693,
      "step": 327
    },
    {
      "epoch": 0.19050938026369285,
      "grad_norm": 313.35858154296875,
      "learning_rate": 4.5235328297501455e-06,
      "logits/chosen": -0.7620694041252136,
      "logits/rejected": -0.7337735295295715,
      "logps/chosen": -68.53215026855469,
      "logps/rejected": -75.737548828125,
      "loss": 13.5514,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0018144982168450952,
      "rewards/margins": 0.035731058567762375,
      "rewards/rejected": -0.03391656279563904,
      "step": 328
    },
    {
      "epoch": 0.1910902015449846,
      "grad_norm": 371.4503479003906,
      "learning_rate": 4.522080185938408e-06,
      "logits/chosen": -0.8343189358711243,
      "logits/rejected": -0.887412428855896,
      "logps/chosen": -69.50465393066406,
      "logps/rejected": -78.40193939208984,
      "loss": 13.6258,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.025033259764313698,
      "rewards/margins": 0.02790558710694313,
      "rewards/rejected": -0.05293884128332138,
      "step": 329
    },
    {
      "epoch": 0.19167102282627635,
      "grad_norm": 368.6702575683594,
      "learning_rate": 4.520627542126671e-06,
      "logits/chosen": -0.8924549221992493,
      "logits/rejected": -0.88835608959198,
      "logps/chosen": -67.6229019165039,
      "logps/rejected": -74.09444427490234,
      "loss": 13.7015,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01569213904440403,
      "rewards/margins": 0.018520962446928024,
      "rewards/rejected": -0.0028288268949836493,
      "step": 330
    },
    {
      "epoch": 0.1922518441075681,
      "grad_norm": 324.3641052246094,
      "learning_rate": 4.519174898314934e-06,
      "logits/chosen": -0.8418426513671875,
      "logits/rejected": -0.8403279185295105,
      "logps/chosen": -72.23628234863281,
      "logps/rejected": -74.98982238769531,
      "loss": 13.8397,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.013874625787138939,
      "rewards/margins": 0.007325878832489252,
      "rewards/rejected": -0.021200504153966904,
      "step": 331
    },
    {
      "epoch": 0.19283266538885985,
      "grad_norm": 321.1864929199219,
      "learning_rate": 4.5177222545031964e-06,
      "logits/chosen": -0.8301184773445129,
      "logits/rejected": -0.748178243637085,
      "logps/chosen": -72.51194763183594,
      "logps/rejected": -72.02323913574219,
      "loss": 13.6885,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.00831710733473301,
      "rewards/margins": 0.020771954208612442,
      "rewards/rejected": -0.02908906899392605,
      "step": 332
    },
    {
      "epoch": 0.1934134866701516,
      "grad_norm": 302.070556640625,
      "learning_rate": 4.516269610691459e-06,
      "logits/chosen": -0.8225947618484497,
      "logits/rejected": -0.7799954414367676,
      "logps/chosen": -73.186279296875,
      "logps/rejected": -67.39868927001953,
      "loss": 14.105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012917375192046165,
      "rewards/margins": -0.019778212532401085,
      "rewards/rejected": 0.006860838737338781,
      "step": 333
    },
    {
      "epoch": 0.19399430795144335,
      "grad_norm": 320.9998779296875,
      "learning_rate": 4.514816966879721e-06,
      "logits/chosen": -0.8660848736763,
      "logits/rejected": -0.8538210988044739,
      "logps/chosen": -77.54426574707031,
      "logps/rejected": -70.16163635253906,
      "loss": 14.0522,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.008558349683880806,
      "rewards/margins": -0.00849790871143341,
      "rewards/rejected": -6.044358087820001e-05,
      "step": 334
    },
    {
      "epoch": 0.1945751292327351,
      "grad_norm": 321.6021423339844,
      "learning_rate": 4.513364323067984e-06,
      "logits/chosen": -0.8346333503723145,
      "logits/rejected": -0.88763827085495,
      "logps/chosen": -74.38595581054688,
      "logps/rejected": -65.50772094726562,
      "loss": 14.0522,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.004940299782902002,
      "rewards/margins": -0.015277815982699394,
      "rewards/rejected": 0.010337515734136105,
      "step": 335
    },
    {
      "epoch": 0.19515595051402684,
      "grad_norm": 329.2897033691406,
      "learning_rate": 4.511911679256247e-06,
      "logits/chosen": -0.9041665196418762,
      "logits/rejected": -0.8650445938110352,
      "logps/chosen": -77.61279296875,
      "logps/rejected": -78.71620178222656,
      "loss": 13.5759,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0006745524588041008,
      "rewards/margins": 0.03203847259283066,
      "rewards/rejected": -0.03136391565203667,
      "step": 336
    },
    {
      "epoch": 0.1957367717953186,
      "grad_norm": 314.5374450683594,
      "learning_rate": 4.510459035444509e-06,
      "logits/chosen": -0.7889400720596313,
      "logits/rejected": -0.7860768437385559,
      "logps/chosen": -73.5044174194336,
      "logps/rejected": -74.55040740966797,
      "loss": 13.7341,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.00419281842187047,
      "rewards/margins": 0.017519472166895866,
      "rewards/rejected": -0.013326652348041534,
      "step": 337
    },
    {
      "epoch": 0.19631759307661031,
      "grad_norm": 303.45733642578125,
      "learning_rate": 4.509006391632772e-06,
      "logits/chosen": -0.9617490768432617,
      "logits/rejected": -0.9624162912368774,
      "logps/chosen": -72.66578674316406,
      "logps/rejected": -73.77256774902344,
      "loss": 13.46,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.024341020733118057,
      "rewards/margins": 0.044455137103796005,
      "rewards/rejected": -0.020114116370677948,
      "step": 338
    },
    {
      "epoch": 0.19689841435790206,
      "grad_norm": 297.1424560546875,
      "learning_rate": 4.507553747821035e-06,
      "logits/chosen": -0.8083046078681946,
      "logits/rejected": -0.8331934213638306,
      "logps/chosen": -71.33175659179688,
      "logps/rejected": -72.65750885009766,
      "loss": 13.8942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017249945551156998,
      "rewards/margins": 0.002586670219898224,
      "rewards/rejected": 0.014663276262581348,
      "step": 339
    },
    {
      "epoch": 0.1974792356391938,
      "grad_norm": 312.68145751953125,
      "learning_rate": 4.506101104009298e-06,
      "logits/chosen": -0.861495316028595,
      "logits/rejected": -0.8806974291801453,
      "logps/chosen": -73.86710357666016,
      "logps/rejected": -74.84933471679688,
      "loss": 13.6776,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.015142458491027355,
      "rewards/margins": 0.02246389351785183,
      "rewards/rejected": -0.03760635107755661,
      "step": 340
    },
    {
      "epoch": 0.19806005692048556,
      "grad_norm": 306.0162353515625,
      "learning_rate": 4.5046484601975595e-06,
      "logits/chosen": -0.7776416540145874,
      "logits/rejected": -0.7664206624031067,
      "logps/chosen": -74.73188018798828,
      "logps/rejected": -71.7195053100586,
      "loss": 13.8777,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01987052895128727,
      "rewards/margins": 0.001284526428207755,
      "rewards/rejected": -0.02115505374968052,
      "step": 341
    },
    {
      "epoch": 0.1986408782017773,
      "grad_norm": 318.2908935546875,
      "learning_rate": 4.503195816385822e-06,
      "logits/chosen": -0.9286810159683228,
      "logits/rejected": -0.8470717668533325,
      "logps/chosen": -77.0329360961914,
      "logps/rejected": -79.6993637084961,
      "loss": 13.9516,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03609747439622879,
      "rewards/margins": -0.0019962512888014317,
      "rewards/rejected": -0.0341012142598629,
      "step": 342
    },
    {
      "epoch": 0.19922169948306906,
      "grad_norm": 343.0354309082031,
      "learning_rate": 4.501743172574085e-06,
      "logits/chosen": -0.6656073331832886,
      "logits/rejected": -0.6898230314254761,
      "logps/chosen": -75.619384765625,
      "logps/rejected": -77.71598052978516,
      "loss": 13.666,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0036884776782244444,
      "rewards/margins": 0.024708479642868042,
      "rewards/rejected": -0.028396958485245705,
      "step": 343
    },
    {
      "epoch": 0.1998025207643608,
      "grad_norm": 318.9757995605469,
      "learning_rate": 4.500290528762348e-06,
      "logits/chosen": -0.9170076251029968,
      "logits/rejected": -0.9125305414199829,
      "logps/chosen": -61.814781188964844,
      "logps/rejected": -73.81932830810547,
      "loss": 13.8999,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.009178831242024899,
      "rewards/margins": 0.0020302850753068924,
      "rewards/rejected": -0.011209115386009216,
      "step": 344
    },
    {
      "epoch": 0.20038334204565256,
      "grad_norm": 301.6319580078125,
      "learning_rate": 4.4988378849506105e-06,
      "logits/chosen": -0.8031547665596008,
      "logits/rejected": -0.8383834958076477,
      "logps/chosen": -71.64036560058594,
      "logps/rejected": -69.63984680175781,
      "loss": 14.3238,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024623652920126915,
      "rewards/margins": -0.042028360068798065,
      "rewards/rejected": 0.0174047090113163,
      "step": 345
    },
    {
      "epoch": 0.2009641633269443,
      "grad_norm": 312.9844055175781,
      "learning_rate": 4.497385241138873e-06,
      "logits/chosen": -0.9153481721878052,
      "logits/rejected": -0.880916953086853,
      "logps/chosen": -75.21138000488281,
      "logps/rejected": -69.07037353515625,
      "loss": 13.5771,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0012881949078291655,
      "rewards/margins": 0.031341440975666046,
      "rewards/rejected": -0.03262963145971298,
      "step": 346
    },
    {
      "epoch": 0.20154498460823606,
      "grad_norm": 349.6585388183594,
      "learning_rate": 4.495932597327136e-06,
      "logits/chosen": -0.7509949803352356,
      "logits/rejected": -0.6910394430160522,
      "logps/chosen": -70.14628601074219,
      "logps/rejected": -71.41559600830078,
      "loss": 14.3833,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.01079236064106226,
      "rewards/margins": -0.048026543110609055,
      "rewards/rejected": 0.03723418340086937,
      "step": 347
    },
    {
      "epoch": 0.2021258058895278,
      "grad_norm": 291.7655029296875,
      "learning_rate": 4.494479953515398e-06,
      "logits/chosen": -0.8026542663574219,
      "logits/rejected": -0.803175151348114,
      "logps/chosen": -71.43840789794922,
      "logps/rejected": -71.1689224243164,
      "loss": 13.6799,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0039477264508605,
      "rewards/margins": 0.021708643063902855,
      "rewards/rejected": -0.01776091754436493,
      "step": 348
    },
    {
      "epoch": 0.20270662717081953,
      "grad_norm": 293.5408935546875,
      "learning_rate": 4.493027309703661e-06,
      "logits/chosen": -0.8886486887931824,
      "logits/rejected": -0.8125056028366089,
      "logps/chosen": -77.56742095947266,
      "logps/rejected": -66.32499694824219,
      "loss": 13.7167,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02104121446609497,
      "rewards/margins": 0.019321372732520103,
      "rewards/rejected": 0.001719839172437787,
      "step": 349
    },
    {
      "epoch": 0.20328744845211127,
      "grad_norm": 305.96697998046875,
      "learning_rate": 4.491574665891923e-06,
      "logits/chosen": -0.7537021636962891,
      "logits/rejected": -0.7224977016448975,
      "logps/chosen": -71.26932525634766,
      "logps/rejected": -69.65036010742188,
      "loss": 13.7664,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.008614836260676384,
      "rewards/margins": 0.012173959985375404,
      "rewards/rejected": -0.02078879624605179,
      "step": 350
    },
    {
      "epoch": 0.20386826973340302,
      "grad_norm": 304.3200988769531,
      "learning_rate": 4.490122022080186e-06,
      "logits/chosen": -0.7840813398361206,
      "logits/rejected": -0.7836223840713501,
      "logps/chosen": -67.91246032714844,
      "logps/rejected": -72.22390747070312,
      "loss": 13.389,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.016956433653831482,
      "rewards/margins": 0.051059335470199585,
      "rewards/rejected": -0.034102894365787506,
      "step": 351
    },
    {
      "epoch": 0.20444909101469477,
      "grad_norm": 291.3819885253906,
      "learning_rate": 4.488669378268449e-06,
      "logits/chosen": -0.7469109296798706,
      "logits/rejected": -0.7488623857498169,
      "logps/chosen": -69.54528045654297,
      "logps/rejected": -72.68141174316406,
      "loss": 13.5492,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.017030106857419014,
      "rewards/margins": 0.03490540385246277,
      "rewards/rejected": -0.017875295132398605,
      "step": 352
    },
    {
      "epoch": 0.20502991229598652,
      "grad_norm": 311.4941711425781,
      "learning_rate": 4.487216734456712e-06,
      "logits/chosen": -0.7565064430236816,
      "logits/rejected": -0.814095675945282,
      "logps/chosen": -77.25660705566406,
      "logps/rejected": -70.50426483154297,
      "loss": 13.9861,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.023350324481725693,
      "rewards/margins": -0.00980983767658472,
      "rewards/rejected": -0.013540486805140972,
      "step": 353
    },
    {
      "epoch": 0.20561073357727827,
      "grad_norm": 317.1282958984375,
      "learning_rate": 4.485764090644974e-06,
      "logits/chosen": -0.6474046111106873,
      "logits/rejected": -0.5795416235923767,
      "logps/chosen": -73.16572570800781,
      "logps/rejected": -73.50746154785156,
      "loss": 13.6889,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.022182445973157883,
      "rewards/margins": 0.024161117151379585,
      "rewards/rejected": -0.0019786744378507137,
      "step": 354
    },
    {
      "epoch": 0.20619155485857002,
      "grad_norm": 353.6116638183594,
      "learning_rate": 4.484311446833236e-06,
      "logits/chosen": -0.6704899072647095,
      "logits/rejected": -0.7246066331863403,
      "logps/chosen": -75.37571716308594,
      "logps/rejected": -77.72221374511719,
      "loss": 14.1555,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.020454417914152145,
      "rewards/margins": -0.026207536458969116,
      "rewards/rejected": 0.005753117147833109,
      "step": 355
    },
    {
      "epoch": 0.20677237613986177,
      "grad_norm": 306.08367919921875,
      "learning_rate": 4.482858803021499e-06,
      "logits/chosen": -0.8278299570083618,
      "logits/rejected": -0.7702374458312988,
      "logps/chosen": -79.121337890625,
      "logps/rejected": -79.2762680053711,
      "loss": 13.3845,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.017661219462752342,
      "rewards/margins": 0.05229003354907036,
      "rewards/rejected": -0.034628815948963165,
      "step": 356
    },
    {
      "epoch": 0.20735319742115352,
      "grad_norm": 301.5957946777344,
      "learning_rate": 4.481406159209763e-06,
      "logits/chosen": -0.7846859693527222,
      "logits/rejected": -0.6674401164054871,
      "logps/chosen": -70.37995910644531,
      "logps/rejected": -72.95897674560547,
      "loss": 13.5715,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.012851124629378319,
      "rewards/margins": 0.033614885061979294,
      "rewards/rejected": -0.020763758569955826,
      "step": 357
    },
    {
      "epoch": 0.20793401870244527,
      "grad_norm": 292.0884094238281,
      "learning_rate": 4.479953515398025e-06,
      "logits/chosen": -0.7394891381263733,
      "logits/rejected": -0.7185255289077759,
      "logps/chosen": -69.58491516113281,
      "logps/rejected": -70.53492736816406,
      "loss": 13.4921,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.008961153216660023,
      "rewards/margins": 0.04007042199373245,
      "rewards/rejected": -0.031109267845749855,
      "step": 358
    },
    {
      "epoch": 0.20851483998373702,
      "grad_norm": 307.0670471191406,
      "learning_rate": 4.478500871586287e-06,
      "logits/chosen": -0.7576676607131958,
      "logits/rejected": -0.8148140907287598,
      "logps/chosen": -73.82097625732422,
      "logps/rejected": -76.19036102294922,
      "loss": 13.6695,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.014283361844718456,
      "rewards/margins": 0.0231223963201046,
      "rewards/rejected": -0.008839035406708717,
      "step": 359
    },
    {
      "epoch": 0.20909566126502874,
      "grad_norm": 300.3365173339844,
      "learning_rate": 4.47704822777455e-06,
      "logits/chosen": -0.8952158093452454,
      "logits/rejected": -0.7517813444137573,
      "logps/chosen": -67.8691635131836,
      "logps/rejected": -75.82969665527344,
      "loss": 14.1246,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.004317197948694229,
      "rewards/margins": -0.022498121485114098,
      "rewards/rejected": 0.01818092353641987,
      "step": 360
    },
    {
      "epoch": 0.2096764825463205,
      "grad_norm": 322.0217590332031,
      "learning_rate": 4.475595583962813e-06,
      "logits/chosen": -0.9054155349731445,
      "logits/rejected": -0.7519516348838806,
      "logps/chosen": -73.81487274169922,
      "logps/rejected": -73.7669906616211,
      "loss": 13.4128,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.025307711213827133,
      "rewards/margins": 0.04915634170174599,
      "rewards/rejected": -0.023848628625273705,
      "step": 361
    },
    {
      "epoch": 0.21025730382761224,
      "grad_norm": 305.5541687011719,
      "learning_rate": 4.4741429401510755e-06,
      "logits/chosen": -0.7097185254096985,
      "logits/rejected": -0.7574303150177002,
      "logps/chosen": -71.90687561035156,
      "logps/rejected": -68.58987426757812,
      "loss": 13.4067,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.00799614004790783,
      "rewards/margins": 0.051417239010334015,
      "rewards/rejected": -0.059413373470306396,
      "step": 362
    },
    {
      "epoch": 0.21083812510890398,
      "grad_norm": 353.3551025390625,
      "learning_rate": 4.472690296339338e-06,
      "logits/chosen": -0.925432026386261,
      "logits/rejected": -0.7938982844352722,
      "logps/chosen": -65.05897521972656,
      "logps/rejected": -73.69389343261719,
      "loss": 13.7132,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.015448915772140026,
      "rewards/margins": 0.017804250121116638,
      "rewards/rejected": -0.03325316682457924,
      "step": 363
    },
    {
      "epoch": 0.21141894639019573,
      "grad_norm": 295.042236328125,
      "learning_rate": 4.471237652527601e-06,
      "logits/chosen": -0.5853012800216675,
      "logits/rejected": -0.6785317659378052,
      "logps/chosen": -70.28089904785156,
      "logps/rejected": -81.96311950683594,
      "loss": 13.2481,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01614985801279545,
      "rewards/margins": 0.06508197635412216,
      "rewards/rejected": -0.04893212765455246,
      "step": 364
    },
    {
      "epoch": 0.21199976767148748,
      "grad_norm": 317.24664306640625,
      "learning_rate": 4.469785008715864e-06,
      "logits/chosen": -0.7670639157295227,
      "logits/rejected": -0.732401967048645,
      "logps/chosen": -73.05061340332031,
      "logps/rejected": -82.63640594482422,
      "loss": 13.4454,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009124317206442356,
      "rewards/margins": 0.047481339424848557,
      "rewards/rejected": -0.05660565569996834,
      "step": 365
    },
    {
      "epoch": 0.21258058895277923,
      "grad_norm": 308.0080871582031,
      "learning_rate": 4.468332364904126e-06,
      "logits/chosen": -0.7194598913192749,
      "logits/rejected": -0.6728766560554504,
      "logps/chosen": -73.11915588378906,
      "logps/rejected": -68.853759765625,
      "loss": 13.9253,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.021355021744966507,
      "rewards/margins": -0.003295204136520624,
      "rewards/rejected": -0.01805981807410717,
      "step": 366
    },
    {
      "epoch": 0.21316141023407098,
      "grad_norm": 327.4511413574219,
      "learning_rate": 4.466879721092388e-06,
      "logits/chosen": -0.7333757281303406,
      "logits/rejected": -0.7388908863067627,
      "logps/chosen": -81.80522155761719,
      "logps/rejected": -69.60041046142578,
      "loss": 13.8993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011159257963299751,
      "rewards/margins": 0.005005787592381239,
      "rewards/rejected": -0.016165047883987427,
      "step": 367
    },
    {
      "epoch": 0.21374223151536273,
      "grad_norm": 298.0936279296875,
      "learning_rate": 4.465427077280651e-06,
      "logits/chosen": -0.9420258402824402,
      "logits/rejected": -0.7203485369682312,
      "logps/chosen": -78.59254455566406,
      "logps/rejected": -70.5693130493164,
      "loss": 13.4892,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02526816353201866,
      "rewards/margins": 0.04102920740842819,
      "rewards/rejected": -0.01576104201376438,
      "step": 368
    },
    {
      "epoch": 0.21432305279665448,
      "grad_norm": 297.84136962890625,
      "learning_rate": 4.463974433468914e-06,
      "logits/chosen": -0.8287162780761719,
      "logits/rejected": -0.8401540517807007,
      "logps/chosen": -70.97822570800781,
      "logps/rejected": -73.44509887695312,
      "loss": 13.4517,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03007066808640957,
      "rewards/margins": 0.047326039522886276,
      "rewards/rejected": -0.017255373299121857,
      "step": 369
    },
    {
      "epoch": 0.21490387407794623,
      "grad_norm": 322.31854248046875,
      "learning_rate": 4.462521789657177e-06,
      "logits/chosen": -0.936688244342804,
      "logits/rejected": -0.8096345663070679,
      "logps/chosen": -78.00273132324219,
      "logps/rejected": -76.44258117675781,
      "loss": 13.933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024149218574166298,
      "rewards/margins": 0.0015145957004278898,
      "rewards/rejected": -0.025663817301392555,
      "step": 370
    },
    {
      "epoch": 0.21548469535923795,
      "grad_norm": 316.048583984375,
      "learning_rate": 4.461069145845439e-06,
      "logits/chosen": -0.7057862877845764,
      "logits/rejected": -0.6469615697860718,
      "logps/chosen": -77.95980834960938,
      "logps/rejected": -83.23860931396484,
      "loss": 13.6137,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.009251176379621029,
      "rewards/margins": 0.03268672153353691,
      "rewards/rejected": -0.04193788766860962,
      "step": 371
    },
    {
      "epoch": 0.2160655166405297,
      "grad_norm": 306.4630432128906,
      "learning_rate": 4.459616502033702e-06,
      "logits/chosen": -0.4769328534603119,
      "logits/rejected": -0.44898924231529236,
      "logps/chosen": -77.04486846923828,
      "logps/rejected": -78.56315612792969,
      "loss": 13.6158,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02271147631108761,
      "rewards/margins": 0.02890445664525032,
      "rewards/rejected": -0.05161593109369278,
      "step": 372
    },
    {
      "epoch": 0.21664633792182145,
      "grad_norm": 304.4960021972656,
      "learning_rate": 4.458163858221964e-06,
      "logits/chosen": -0.7167657017707825,
      "logits/rejected": -0.6918506622314453,
      "logps/chosen": -71.01347351074219,
      "logps/rejected": -72.31254577636719,
      "loss": 13.732,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.000307169568259269,
      "rewards/margins": 0.01947268471121788,
      "rewards/rejected": -0.0191655196249485,
      "step": 373
    },
    {
      "epoch": 0.2172271592031132,
      "grad_norm": 296.60205078125,
      "learning_rate": 4.456711214410227e-06,
      "logits/chosen": -0.7357224225997925,
      "logits/rejected": -0.6462749242782593,
      "logps/chosen": -75.4621810913086,
      "logps/rejected": -70.76265716552734,
      "loss": 14.1072,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03969653695821762,
      "rewards/margins": -0.01930340752005577,
      "rewards/rejected": -0.02039313316345215,
      "step": 374
    },
    {
      "epoch": 0.21780798048440494,
      "grad_norm": 299.0235595703125,
      "learning_rate": 4.4552585705984895e-06,
      "logits/chosen": -0.9737260937690735,
      "logits/rejected": -0.883916974067688,
      "logps/chosen": -76.01069641113281,
      "logps/rejected": -64.29400634765625,
      "loss": 13.8576,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0021567821968346834,
      "rewards/margins": 0.005682178307324648,
      "rewards/rejected": -0.003525395644828677,
      "step": 375
    },
    {
      "epoch": 0.2183888017656967,
      "grad_norm": 323.23663330078125,
      "learning_rate": 4.453805926786752e-06,
      "logits/chosen": -0.7908447980880737,
      "logits/rejected": -0.7856351137161255,
      "logps/chosen": -73.55455017089844,
      "logps/rejected": -75.9771957397461,
      "loss": 14.1184,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.018270045518875122,
      "rewards/margins": -0.020638611167669296,
      "rewards/rejected": 0.0023685642518103123,
      "step": 376
    },
    {
      "epoch": 0.21896962304698844,
      "grad_norm": 322.6252746582031,
      "learning_rate": 4.452353282975015e-06,
      "logits/chosen": -0.8486648797988892,
      "logits/rejected": -0.8361449241638184,
      "logps/chosen": -71.75032043457031,
      "logps/rejected": -70.97474670410156,
      "loss": 13.4519,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0086447075009346,
      "rewards/margins": 0.04460224509239197,
      "rewards/rejected": -0.05324694514274597,
      "step": 377
    },
    {
      "epoch": 0.2195504443282802,
      "grad_norm": 533.611083984375,
      "learning_rate": 4.450900639163278e-06,
      "logits/chosen": -0.7714306116104126,
      "logits/rejected": -0.7245787382125854,
      "logps/chosen": -73.3573226928711,
      "logps/rejected": -72.71539306640625,
      "loss": 13.7809,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01556567382067442,
      "rewards/margins": 0.01385035365819931,
      "rewards/rejected": -0.029416028410196304,
      "step": 378
    },
    {
      "epoch": 0.22013126560957194,
      "grad_norm": 287.4798583984375,
      "learning_rate": 4.4494479953515405e-06,
      "logits/chosen": -0.8422917127609253,
      "logits/rejected": -0.7723952531814575,
      "logps/chosen": -67.02293395996094,
      "logps/rejected": -76.67664337158203,
      "loss": 13.2566,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.013728572055697441,
      "rewards/margins": 0.0658256933093071,
      "rewards/rejected": -0.052097123116254807,
      "step": 379
    },
    {
      "epoch": 0.2207120868908637,
      "grad_norm": 333.2846374511719,
      "learning_rate": 4.4479953515398024e-06,
      "logits/chosen": -0.7608442306518555,
      "logits/rejected": -0.7422316670417786,
      "logps/chosen": -72.54397583007812,
      "logps/rejected": -69.54707336425781,
      "loss": 14.3318,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.04744737595319748,
      "rewards/margins": -0.040470145642757416,
      "rewards/rejected": -0.00697722565382719,
      "step": 380
    },
    {
      "epoch": 0.22129290817215544,
      "grad_norm": 315.6219177246094,
      "learning_rate": 4.446542707728065e-06,
      "logits/chosen": -0.7969782948493958,
      "logits/rejected": -0.7673382759094238,
      "logps/chosen": -76.81396484375,
      "logps/rejected": -74.59119415283203,
      "loss": 14.2554,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04644475132226944,
      "rewards/margins": -0.03383153676986694,
      "rewards/rejected": -0.012613209895789623,
      "step": 381
    },
    {
      "epoch": 0.22187372945344716,
      "grad_norm": 360.37274169921875,
      "learning_rate": 4.445090063916328e-06,
      "logits/chosen": -0.7555001974105835,
      "logits/rejected": -0.8732713460922241,
      "logps/chosen": -80.61076354980469,
      "logps/rejected": -67.64311981201172,
      "loss": 14.0055,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.014537744224071503,
      "rewards/margins": -0.010739547200500965,
      "rewards/rejected": -0.0037982002831995487,
      "step": 382
    },
    {
      "epoch": 0.2224545507347389,
      "grad_norm": 305.7255554199219,
      "learning_rate": 4.443637420104591e-06,
      "logits/chosen": -0.6732084155082703,
      "logits/rejected": -0.6309961080551147,
      "logps/chosen": -69.0905532836914,
      "logps/rejected": -72.41564178466797,
      "loss": 13.2323,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.053167276084423065,
      "rewards/margins": 0.06811966747045517,
      "rewards/rejected": -0.014952393248677254,
      "step": 383
    },
    {
      "epoch": 0.22303537201603066,
      "grad_norm": 318.80755615234375,
      "learning_rate": 4.4421847762928534e-06,
      "logits/chosen": -0.8548835515975952,
      "logits/rejected": -0.7376150488853455,
      "logps/chosen": -74.19415283203125,
      "logps/rejected": -83.93922424316406,
      "loss": 14.1736,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.04532115161418915,
      "rewards/margins": -0.02556682750582695,
      "rewards/rejected": -0.01975431852042675,
      "step": 384
    },
    {
      "epoch": 0.2236161932973224,
      "grad_norm": 298.5427551269531,
      "learning_rate": 4.440732132481116e-06,
      "logits/chosen": -0.7442782521247864,
      "logits/rejected": -0.7569425702095032,
      "logps/chosen": -70.04218292236328,
      "logps/rejected": -76.72520446777344,
      "loss": 13.364,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.015433462336659431,
      "rewards/margins": 0.052092112600803375,
      "rewards/rejected": -0.03665865212678909,
      "step": 385
    },
    {
      "epoch": 0.22419701457861416,
      "grad_norm": 309.5966491699219,
      "learning_rate": 4.439279488669379e-06,
      "logits/chosen": -0.6254906058311462,
      "logits/rejected": -0.6205364465713501,
      "logps/chosen": -72.59516906738281,
      "logps/rejected": -75.21646881103516,
      "loss": 13.6555,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.004701803904026747,
      "rewards/margins": 0.02681594528257847,
      "rewards/rejected": -0.031517744064331055,
      "step": 386
    },
    {
      "epoch": 0.2247778358599059,
      "grad_norm": 288.473876953125,
      "learning_rate": 4.437826844857641e-06,
      "logits/chosen": -0.8325392603874207,
      "logits/rejected": -1.0077170133590698,
      "logps/chosen": -72.17142486572266,
      "logps/rejected": -77.3849105834961,
      "loss": 13.1622,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.030657533556222916,
      "rewards/margins": 0.08042553812265396,
      "rewards/rejected": -0.049768008291721344,
      "step": 387
    },
    {
      "epoch": 0.22535865714119765,
      "grad_norm": 337.0882263183594,
      "learning_rate": 4.436374201045904e-06,
      "logits/chosen": -0.6630915999412537,
      "logits/rejected": -0.7251507639884949,
      "logps/chosen": -79.76948547363281,
      "logps/rejected": -77.0470199584961,
      "loss": 14.1512,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01571335829794407,
      "rewards/margins": -0.025171738117933273,
      "rewards/rejected": 0.009458379819989204,
      "step": 388
    },
    {
      "epoch": 0.2259394784224894,
      "grad_norm": 312.39605712890625,
      "learning_rate": 4.434921557234166e-06,
      "logits/chosen": -0.7051008343696594,
      "logits/rejected": -0.7591395974159241,
      "logps/chosen": -70.51537322998047,
      "logps/rejected": -74.8718032836914,
      "loss": 14.1159,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.05321550369262695,
      "rewards/margins": -0.019576644524931908,
      "rewards/rejected": -0.033638857305049896,
      "step": 389
    },
    {
      "epoch": 0.22652029970378115,
      "grad_norm": 318.12457275390625,
      "learning_rate": 4.433468913422429e-06,
      "logits/chosen": -0.7092422246932983,
      "logits/rejected": -0.7855595946311951,
      "logps/chosen": -73.6144790649414,
      "logps/rejected": -70.25662231445312,
      "loss": 13.7536,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015159961767494678,
      "rewards/margins": 0.01655631884932518,
      "rewards/rejected": -0.03171628341078758,
      "step": 390
    },
    {
      "epoch": 0.2271011209850729,
      "grad_norm": 297.23565673828125,
      "learning_rate": 4.432016269610692e-06,
      "logits/chosen": -0.6911486387252808,
      "logits/rejected": -0.5650784373283386,
      "logps/chosen": -61.7591438293457,
      "logps/rejected": -79.15550231933594,
      "loss": 13.4395,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.007602124474942684,
      "rewards/margins": 0.0479809045791626,
      "rewards/rejected": -0.04037877544760704,
      "step": 391
    },
    {
      "epoch": 0.22768194226636465,
      "grad_norm": 315.382080078125,
      "learning_rate": 4.4305636257989546e-06,
      "logits/chosen": -0.7994186282157898,
      "logits/rejected": -0.8166016340255737,
      "logps/chosen": -81.44297790527344,
      "logps/rejected": -74.20890808105469,
      "loss": 13.8923,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.06704780459403992,
      "rewards/margins": 0.012969402596354485,
      "rewards/rejected": -0.08001720905303955,
      "step": 392
    },
    {
      "epoch": 0.2282627635476564,
      "grad_norm": 310.18865966796875,
      "learning_rate": 4.4291109819872165e-06,
      "logits/chosen": -0.6670494079589844,
      "logits/rejected": -0.7397955656051636,
      "logps/chosen": -69.39032745361328,
      "logps/rejected": -77.76183319091797,
      "loss": 13.8815,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.033610161393880844,
      "rewards/margins": 0.0040161325596272945,
      "rewards/rejected": -0.037626296281814575,
      "step": 393
    },
    {
      "epoch": 0.22884358482894812,
      "grad_norm": 1028.6781005859375,
      "learning_rate": 4.427658338175479e-06,
      "logits/chosen": -0.6542873978614807,
      "logits/rejected": -0.6277574300765991,
      "logps/chosen": -71.77285766601562,
      "logps/rejected": -82.8670654296875,
      "loss": 13.6692,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.003381280694156885,
      "rewards/margins": 0.02418811246752739,
      "rewards/rejected": -0.02756938897073269,
      "step": 394
    },
    {
      "epoch": 0.22942440611023987,
      "grad_norm": 300.6793212890625,
      "learning_rate": 4.426205694363742e-06,
      "logits/chosen": -0.657829761505127,
      "logits/rejected": -0.6191960573196411,
      "logps/chosen": -71.67803192138672,
      "logps/rejected": -71.7804183959961,
      "loss": 13.7474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0207084771245718,
      "rewards/margins": 0.01424718089401722,
      "rewards/rejected": 0.006461297161877155,
      "step": 395
    },
    {
      "epoch": 0.23000522739153162,
      "grad_norm": 336.28759765625,
      "learning_rate": 4.424753050552005e-06,
      "logits/chosen": -0.7785569429397583,
      "logits/rejected": -0.8487392663955688,
      "logps/chosen": -82.74131774902344,
      "logps/rejected": -74.78742980957031,
      "loss": 14.0335,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03686489909887314,
      "rewards/margins": -0.003692910075187683,
      "rewards/rejected": -0.03317200019955635,
      "step": 396
    },
    {
      "epoch": 0.23058604867282337,
      "grad_norm": 300.73101806640625,
      "learning_rate": 4.4233004067402675e-06,
      "logits/chosen": -0.7787965536117554,
      "logits/rejected": -0.7510574460029602,
      "logps/chosen": -72.66011047363281,
      "logps/rejected": -75.55070495605469,
      "loss": 13.6585,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.015632599592208862,
      "rewards/margins": 0.02679685689508915,
      "rewards/rejected": -0.04242945462465286,
      "step": 397
    },
    {
      "epoch": 0.23116686995411512,
      "grad_norm": 297.7178955078125,
      "learning_rate": 4.42184776292853e-06,
      "logits/chosen": -0.6219455003738403,
      "logits/rejected": -0.619706928730011,
      "logps/chosen": -72.49166107177734,
      "logps/rejected": -64.84117126464844,
      "loss": 13.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011766968294978142,
      "rewards/margins": 0.021194420754909515,
      "rewards/rejected": -0.03296138718724251,
      "step": 398
    },
    {
      "epoch": 0.23174769123540687,
      "grad_norm": 1181.5919189453125,
      "learning_rate": 4.420395119116793e-06,
      "logits/chosen": -0.7124764323234558,
      "logits/rejected": -0.7599982023239136,
      "logps/chosen": -82.6731185913086,
      "logps/rejected": -77.68875885009766,
      "loss": 13.8524,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004232340957969427,
      "rewards/margins": 0.005291967652738094,
      "rewards/rejected": -0.001059626811183989,
      "step": 399
    },
    {
      "epoch": 0.23232851251669862,
      "grad_norm": 291.1778869628906,
      "learning_rate": 4.418942475305056e-06,
      "logits/chosen": -0.707445502281189,
      "logits/rejected": -0.7544277310371399,
      "logps/chosen": -67.69174194335938,
      "logps/rejected": -73.61653137207031,
      "loss": 13.4031,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02160044014453888,
      "rewards/margins": 0.05302933603525162,
      "rewards/rejected": -0.03142889216542244,
      "step": 400
    },
    {
      "epoch": 0.23290933379799036,
      "grad_norm": 308.5126037597656,
      "learning_rate": 4.4174898314933185e-06,
      "logits/chosen": -0.7360697984695435,
      "logits/rejected": -0.6314154863357544,
      "logps/chosen": -73.9341049194336,
      "logps/rejected": -75.5310287475586,
      "loss": 13.7204,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -7.766112685203552e-05,
      "rewards/margins": 0.02340327948331833,
      "rewards/rejected": -0.023480940610170364,
      "step": 401
    },
    {
      "epoch": 0.2334901550792821,
      "grad_norm": 297.6388244628906,
      "learning_rate": 4.416037187681581e-06,
      "logits/chosen": -0.6773185729980469,
      "logits/rejected": -0.6966699957847595,
      "logps/chosen": -71.93721008300781,
      "logps/rejected": -73.58920288085938,
      "loss": 13.8046,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.014304883778095245,
      "rewards/margins": 0.012389096431434155,
      "rewards/rejected": 0.0019157860660925508,
      "step": 402
    },
    {
      "epoch": 0.23407097636057386,
      "grad_norm": 319.0065612792969,
      "learning_rate": 4.414584543869844e-06,
      "logits/chosen": -0.6499051451683044,
      "logits/rejected": -0.9121743440628052,
      "logps/chosen": -81.61251831054688,
      "logps/rejected": -78.27536010742188,
      "loss": 13.3141,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0065872580744326115,
      "rewards/margins": 0.07194839417934418,
      "rewards/rejected": -0.07853565365076065,
      "step": 403
    },
    {
      "epoch": 0.2346517976418656,
      "grad_norm": 398.7077941894531,
      "learning_rate": 4.413131900058107e-06,
      "logits/chosen": -0.6521409749984741,
      "logits/rejected": -0.7569887638092041,
      "logps/chosen": -71.24628448486328,
      "logps/rejected": -66.81242370605469,
      "loss": 13.7869,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0004381166654638946,
      "rewards/margins": 0.012258688919246197,
      "rewards/rejected": -0.011820574291050434,
      "step": 404
    },
    {
      "epoch": 0.23523261892315733,
      "grad_norm": 296.407470703125,
      "learning_rate": 4.411679256246369e-06,
      "logits/chosen": -0.8323850631713867,
      "logits/rejected": -0.7685847282409668,
      "logps/chosen": -70.29356384277344,
      "logps/rejected": -73.77702331542969,
      "loss": 13.6071,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.008858191780745983,
      "rewards/margins": 0.034311629831790924,
      "rewards/rejected": -0.025453437119722366,
      "step": 405
    },
    {
      "epoch": 0.23581344020444908,
      "grad_norm": 316.6470031738281,
      "learning_rate": 4.410226612434631e-06,
      "logits/chosen": -0.8853529691696167,
      "logits/rejected": -0.8087629079818726,
      "logps/chosen": -74.18810272216797,
      "logps/rejected": -75.40741729736328,
      "loss": 13.8599,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02046876773238182,
      "rewards/margins": 0.006527154240757227,
      "rewards/rejected": 0.013941613025963306,
      "step": 406
    },
    {
      "epoch": 0.23639426148574083,
      "grad_norm": 307.0223388671875,
      "learning_rate": 4.408773968622894e-06,
      "logits/chosen": -0.5616191625595093,
      "logits/rejected": -0.5830402970314026,
      "logps/chosen": -73.06590270996094,
      "logps/rejected": -75.1976318359375,
      "loss": 13.5981,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.030349424108862877,
      "rewards/margins": 0.031421925872564316,
      "rewards/rejected": -0.0010725029278546572,
      "step": 407
    },
    {
      "epoch": 0.23697508276703258,
      "grad_norm": 306.7025146484375,
      "learning_rate": 4.407321324811157e-06,
      "logits/chosen": -0.6714197993278503,
      "logits/rejected": -0.7746154069900513,
      "logps/chosen": -76.53195190429688,
      "logps/rejected": -69.44393920898438,
      "loss": 13.6952,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0025624283589422703,
      "rewards/margins": 0.02050391212105751,
      "rewards/rejected": -0.023066340014338493,
      "step": 408
    },
    {
      "epoch": 0.23755590404832433,
      "grad_norm": 315.3130187988281,
      "learning_rate": 4.40586868099942e-06,
      "logits/chosen": -0.6545313000679016,
      "logits/rejected": -0.6212127804756165,
      "logps/chosen": -75.73979187011719,
      "logps/rejected": -75.5727310180664,
      "loss": 13.0526,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.03893669322133064,
      "rewards/margins": 0.08733826875686646,
      "rewards/rejected": -0.04840157553553581,
      "step": 409
    },
    {
      "epoch": 0.23813672532961608,
      "grad_norm": 319.98065185546875,
      "learning_rate": 4.404416037187682e-06,
      "logits/chosen": -0.6923006772994995,
      "logits/rejected": -0.5738077163696289,
      "logps/chosen": -72.50543975830078,
      "logps/rejected": -73.67253112792969,
      "loss": 13.8894,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.011189279146492481,
      "rewards/margins": 0.001272419816814363,
      "rewards/rejected": -0.012461700476706028,
      "step": 410
    },
    {
      "epoch": 0.23871754661090783,
      "grad_norm": 353.4020080566406,
      "learning_rate": 4.402963393375945e-06,
      "logits/chosen": -0.6882558465003967,
      "logits/rejected": -0.6723198890686035,
      "logps/chosen": -75.6532211303711,
      "logps/rejected": -70.86685943603516,
      "loss": 13.8372,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0034420304000377655,
      "rewards/margins": 0.008326428011059761,
      "rewards/rejected": -0.004884395748376846,
      "step": 411
    },
    {
      "epoch": 0.23929836789219958,
      "grad_norm": 305.5731506347656,
      "learning_rate": 4.401510749564207e-06,
      "logits/chosen": -0.7900495529174805,
      "logits/rejected": -0.7300332188606262,
      "logps/chosen": -69.94303894042969,
      "logps/rejected": -73.22996520996094,
      "loss": 13.4269,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.033657319843769073,
      "rewards/margins": 0.048354774713516235,
      "rewards/rejected": -0.014697456732392311,
      "step": 412
    },
    {
      "epoch": 0.23987918917349133,
      "grad_norm": 337.9052734375,
      "learning_rate": 4.40005810575247e-06,
      "logits/chosen": -0.7297667264938354,
      "logits/rejected": -0.8206753730773926,
      "logps/chosen": -79.34073638916016,
      "logps/rejected": -79.64215850830078,
      "loss": 13.3179,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.033630721271038055,
      "rewards/margins": 0.06482113897800446,
      "rewards/rejected": -0.03119041956961155,
      "step": 413
    },
    {
      "epoch": 0.24046001045478307,
      "grad_norm": 328.78985595703125,
      "learning_rate": 4.3986054619407325e-06,
      "logits/chosen": -0.5856087803840637,
      "logits/rejected": -0.6253767013549805,
      "logps/chosen": -76.09996032714844,
      "logps/rejected": -76.34832763671875,
      "loss": 14.0925,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.030036872252821922,
      "rewards/margins": -0.014562776312232018,
      "rewards/rejected": -0.015474090352654457,
      "step": 414
    },
    {
      "epoch": 0.24104083173607482,
      "grad_norm": 297.6679382324219,
      "learning_rate": 4.397152818128995e-06,
      "logits/chosen": -0.8083240389823914,
      "logits/rejected": -0.7946100234985352,
      "logps/chosen": -72.3720703125,
      "logps/rejected": -68.49833679199219,
      "loss": 14.016,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.012605169788002968,
      "rewards/margins": -0.009788742288947105,
      "rewards/rejected": -0.0028164288960397243,
      "step": 415
    },
    {
      "epoch": 0.24162165301736654,
      "grad_norm": 310.3009338378906,
      "learning_rate": 4.395700174317258e-06,
      "logits/chosen": -0.6257360577583313,
      "logits/rejected": -0.6403561234474182,
      "logps/chosen": -76.71080017089844,
      "logps/rejected": -77.48155212402344,
      "loss": 13.5,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04251420125365257,
      "rewards/margins": 0.04183372110128403,
      "rewards/rejected": 0.0006804756703786552,
      "step": 416
    },
    {
      "epoch": 0.2422024742986583,
      "grad_norm": 325.7873840332031,
      "learning_rate": 4.394247530505521e-06,
      "logits/chosen": -0.6726334691047668,
      "logits/rejected": -0.6855700016021729,
      "logps/chosen": -75.29472351074219,
      "logps/rejected": -71.83308410644531,
      "loss": 14.0017,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.018920384347438812,
      "rewards/margins": -0.009463165886700153,
      "rewards/rejected": 0.02838354930281639,
      "step": 417
    },
    {
      "epoch": 0.24278329557995004,
      "grad_norm": 542.8521728515625,
      "learning_rate": 4.3927948866937835e-06,
      "logits/chosen": -0.5671173334121704,
      "logits/rejected": -0.5828499794006348,
      "logps/chosen": -80.865966796875,
      "logps/rejected": -77.36320495605469,
      "loss": 14.1966,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.010454867966473103,
      "rewards/margins": -0.0281839556992054,
      "rewards/rejected": 0.01772909425199032,
      "step": 418
    },
    {
      "epoch": 0.2433641168612418,
      "grad_norm": 313.10650634765625,
      "learning_rate": 4.391342242882045e-06,
      "logits/chosen": -0.5604667663574219,
      "logits/rejected": -0.6470273733139038,
      "logps/chosen": -72.81732177734375,
      "logps/rejected": -70.16290283203125,
      "loss": 13.2515,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017285270616412163,
      "rewards/margins": 0.07715705037117004,
      "rewards/rejected": -0.05987178534269333,
      "step": 419
    },
    {
      "epoch": 0.24394493814253354,
      "grad_norm": 322.7060241699219,
      "learning_rate": 4.389889599070308e-06,
      "logits/chosen": -0.7095221281051636,
      "logits/rejected": -0.70353764295578,
      "logps/chosen": -72.71652221679688,
      "logps/rejected": -67.59053802490234,
      "loss": 14.0589,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017042094841599464,
      "rewards/margins": -0.016451817005872726,
      "rewards/rejected": -0.0005902774864807725,
      "step": 420
    },
    {
      "epoch": 0.2445257594238253,
      "grad_norm": 285.8162536621094,
      "learning_rate": 4.388436955258571e-06,
      "logits/chosen": -0.8471421003341675,
      "logits/rejected": -0.7050653696060181,
      "logps/chosen": -71.2718505859375,
      "logps/rejected": -66.46906280517578,
      "loss": 13.5971,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004547786898910999,
      "rewards/margins": 0.030440161004662514,
      "rewards/rejected": -0.02589237317442894,
      "step": 421
    },
    {
      "epoch": 0.24510658070511704,
      "grad_norm": 312.5526428222656,
      "learning_rate": 4.386984311446834e-06,
      "logits/chosen": -0.6499220728874207,
      "logits/rejected": -0.7068791389465332,
      "logps/chosen": -70.94161987304688,
      "logps/rejected": -67.37171936035156,
      "loss": 13.7637,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.014181653037667274,
      "rewards/margins": 0.019841155037283897,
      "rewards/rejected": -0.005659504793584347,
      "step": 422
    },
    {
      "epoch": 0.2456874019864088,
      "grad_norm": 306.18194580078125,
      "learning_rate": 4.385531667635096e-06,
      "logits/chosen": -0.6181563138961792,
      "logits/rejected": -0.7241290807723999,
      "logps/chosen": -63.1754035949707,
      "logps/rejected": -76.5521469116211,
      "loss": 13.3346,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06029351428151131,
      "rewards/margins": 0.05993475764989853,
      "rewards/rejected": 0.00035875439061783254,
      "step": 423
    },
    {
      "epoch": 0.24626822326770054,
      "grad_norm": 307.78765869140625,
      "learning_rate": 4.384079023823359e-06,
      "logits/chosen": -0.488178014755249,
      "logits/rejected": -0.6498016119003296,
      "logps/chosen": -70.04771423339844,
      "logps/rejected": -67.56314849853516,
      "loss": 13.8241,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.002901268657296896,
      "rewards/margins": 0.014369833283126354,
      "rewards/rejected": -0.011468565091490746,
      "step": 424
    },
    {
      "epoch": 0.24684904454899229,
      "grad_norm": 321.4908142089844,
      "learning_rate": 4.382626380011621e-06,
      "logits/chosen": -0.7453001737594604,
      "logits/rejected": -0.7693713903427124,
      "logps/chosen": -69.00504302978516,
      "logps/rejected": -69.38874816894531,
      "loss": 14.0894,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03128911182284355,
      "rewards/margins": -0.019808074459433556,
      "rewards/rejected": -0.011481037363409996,
      "step": 425
    },
    {
      "epoch": 0.24742986583028403,
      "grad_norm": 306.6564025878906,
      "learning_rate": 4.381173736199884e-06,
      "logits/chosen": -0.6882332563400269,
      "logits/rejected": -0.666528582572937,
      "logps/chosen": -71.15715026855469,
      "logps/rejected": -72.64927673339844,
      "loss": 13.5929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.045943137258291245,
      "rewards/margins": 0.03135029971599579,
      "rewards/rejected": 0.01459283847361803,
      "step": 426
    },
    {
      "epoch": 0.24801068711157576,
      "grad_norm": 319.85552978515625,
      "learning_rate": 4.3797210923881465e-06,
      "logits/chosen": -0.6911331415176392,
      "logits/rejected": -0.6865822076797485,
      "logps/chosen": -70.95181274414062,
      "logps/rejected": -69.65785217285156,
      "loss": 13.6944,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.00031525566009804606,
      "rewards/margins": 0.025617409497499466,
      "rewards/rejected": -0.025302153080701828,
      "step": 427
    },
    {
      "epoch": 0.2485915083928675,
      "grad_norm": 305.7183532714844,
      "learning_rate": 4.378268448576409e-06,
      "logits/chosen": -0.5291553735733032,
      "logits/rejected": -0.5490658283233643,
      "logps/chosen": -69.1864242553711,
      "logps/rejected": -74.73960876464844,
      "loss": 13.8674,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.005592182278633118,
      "rewards/margins": 0.003548829350620508,
      "rewards/rejected": 0.002043351763859391,
      "step": 428
    },
    {
      "epoch": 0.24917232967415925,
      "grad_norm": 297.0389404296875,
      "learning_rate": 4.376815804764672e-06,
      "logits/chosen": -0.6868799924850464,
      "logits/rejected": -0.71811842918396,
      "logps/chosen": -67.46659088134766,
      "logps/rejected": -66.69960021972656,
      "loss": 13.6168,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0253940187394619,
      "rewards/margins": 0.02877502143383026,
      "rewards/rejected": -0.0033809959422796965,
      "step": 429
    },
    {
      "epoch": 0.249753150955451,
      "grad_norm": 304.1397705078125,
      "learning_rate": 4.375363160952935e-06,
      "logits/chosen": -0.6851626634597778,
      "logits/rejected": -0.7611708641052246,
      "logps/chosen": -66.39443969726562,
      "logps/rejected": -74.42601013183594,
      "loss": 14.2112,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.003943081013858318,
      "rewards/margins": -0.02984614297747612,
      "rewards/rejected": 0.03378922492265701,
      "step": 430
    },
    {
      "epoch": 0.25033397223674275,
      "grad_norm": 294.4915771484375,
      "learning_rate": 4.3739105171411975e-06,
      "logits/chosen": -0.6287668943405151,
      "logits/rejected": -0.6079251170158386,
      "logps/chosen": -75.12745666503906,
      "logps/rejected": -70.1561508178711,
      "loss": 13.7044,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.015175809152424335,
      "rewards/margins": 0.02054971642792225,
      "rewards/rejected": -0.005373907275497913,
      "step": 431
    },
    {
      "epoch": 0.2509147935180345,
      "grad_norm": 331.4616394042969,
      "learning_rate": 4.3724578733294594e-06,
      "logits/chosen": -0.6616209149360657,
      "logits/rejected": -0.6820401549339294,
      "logps/chosen": -65.7193374633789,
      "logps/rejected": -67.92347717285156,
      "loss": 13.6835,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.032937806099653244,
      "rewards/margins": 0.024601612240076065,
      "rewards/rejected": 0.008336210623383522,
      "step": 432
    },
    {
      "epoch": 0.25149561479932625,
      "grad_norm": 302.4757385253906,
      "learning_rate": 4.371005229517722e-06,
      "logits/chosen": -0.5871397852897644,
      "logits/rejected": -0.6894339323043823,
      "logps/chosen": -70.20496368408203,
      "logps/rejected": -75.09275817871094,
      "loss": 13.6636,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0664600282907486,
      "rewards/margins": 0.022688765078783035,
      "rewards/rejected": 0.04377124831080437,
      "step": 433
    },
    {
      "epoch": 0.25207643608061797,
      "grad_norm": 301.0609436035156,
      "learning_rate": 4.369552585705985e-06,
      "logits/chosen": -0.72092604637146,
      "logits/rejected": -0.7814493775367737,
      "logps/chosen": -69.15807342529297,
      "logps/rejected": -69.16731262207031,
      "loss": 13.3789,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.03117241896688938,
      "rewards/margins": 0.05374479293823242,
      "rewards/rejected": -0.02257237769663334,
      "step": 434
    },
    {
      "epoch": 0.25265725736190975,
      "grad_norm": 310.7433776855469,
      "learning_rate": 4.368099941894248e-06,
      "logits/chosen": -0.5796935558319092,
      "logits/rejected": -0.6400626301765442,
      "logps/chosen": -74.19486236572266,
      "logps/rejected": -78.7645263671875,
      "loss": 13.3789,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.040846049785614014,
      "rewards/margins": 0.05530615895986557,
      "rewards/rejected": -0.014460104517638683,
      "step": 435
    },
    {
      "epoch": 0.25323807864320147,
      "grad_norm": 320.4769287109375,
      "learning_rate": 4.36664729808251e-06,
      "logits/chosen": -0.7232745289802551,
      "logits/rejected": -0.850020706653595,
      "logps/chosen": -66.91292572021484,
      "logps/rejected": -63.138587951660156,
      "loss": 14.3721,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.009416877292096615,
      "rewards/margins": -0.04486392065882683,
      "rewards/rejected": 0.03544704243540764,
      "step": 436
    },
    {
      "epoch": 0.25381889992449325,
      "grad_norm": 322.4789733886719,
      "learning_rate": 4.365194654270773e-06,
      "logits/chosen": -0.7586982846260071,
      "logits/rejected": -0.8501695394515991,
      "logps/chosen": -74.4521255493164,
      "logps/rejected": -72.19835662841797,
      "loss": 13.5591,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.026751860976219177,
      "rewards/margins": 0.03630850464105606,
      "rewards/rejected": -0.009556648321449757,
      "step": 437
    },
    {
      "epoch": 0.25439972120578497,
      "grad_norm": 301.9642639160156,
      "learning_rate": 4.363742010459036e-06,
      "logits/chosen": -0.7936211228370667,
      "logits/rejected": -0.8182106018066406,
      "logps/chosen": -68.75806427001953,
      "logps/rejected": -67.1334457397461,
      "loss": 13.7116,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03210698813199997,
      "rewards/margins": 0.019230013713240623,
      "rewards/rejected": 0.012876978144049644,
      "step": 438
    },
    {
      "epoch": 0.25498054248707674,
      "grad_norm": 317.6552429199219,
      "learning_rate": 4.362289366647298e-06,
      "logits/chosen": -0.6283025741577148,
      "logits/rejected": -0.5947784781455994,
      "logps/chosen": -72.60492706298828,
      "logps/rejected": -81.98297882080078,
      "loss": 13.6121,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.029752427712082863,
      "rewards/margins": 0.03360595554113388,
      "rewards/rejected": -0.003853529691696167,
      "step": 439
    },
    {
      "epoch": 0.25556136376836847,
      "grad_norm": 301.3587951660156,
      "learning_rate": 4.3608367228355606e-06,
      "logits/chosen": -0.521526575088501,
      "logits/rejected": -0.6442473530769348,
      "logps/chosen": -72.14176177978516,
      "logps/rejected": -79.62040710449219,
      "loss": 13.238,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06225038319826126,
      "rewards/margins": 0.06984097510576248,
      "rewards/rejected": -0.007590600289404392,
      "step": 440
    },
    {
      "epoch": 0.25614218504966024,
      "grad_norm": 301.3314514160156,
      "learning_rate": 4.359384079023824e-06,
      "logits/chosen": -0.5870726108551025,
      "logits/rejected": -0.561357855796814,
      "logps/chosen": -69.20145416259766,
      "logps/rejected": -71.50432586669922,
      "loss": 13.8883,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.029437389224767685,
      "rewards/margins": 0.002829763339832425,
      "rewards/rejected": 0.026607628911733627,
      "step": 441
    },
    {
      "epoch": 0.25672300633095196,
      "grad_norm": 300.83056640625,
      "learning_rate": 4.357931435212087e-06,
      "logits/chosen": -0.6320289969444275,
      "logits/rejected": -0.6687763333320618,
      "logps/chosen": -69.56461334228516,
      "logps/rejected": -73.77947998046875,
      "loss": 13.5317,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04403435066342354,
      "rewards/margins": 0.036295950412750244,
      "rewards/rejected": 0.007738398853689432,
      "step": 442
    },
    {
      "epoch": 0.2573038276122437,
      "grad_norm": 299.238525390625,
      "learning_rate": 4.35647879140035e-06,
      "logits/chosen": -0.7340787053108215,
      "logits/rejected": -0.8285503387451172,
      "logps/chosen": -69.81163024902344,
      "logps/rejected": -69.9961929321289,
      "loss": 13.6267,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.020665764808654785,
      "rewards/margins": 0.035551171749830246,
      "rewards/rejected": -0.014885407872498035,
      "step": 443
    },
    {
      "epoch": 0.25788464889353546,
      "grad_norm": 332.7782897949219,
      "learning_rate": 4.3550261475886116e-06,
      "logits/chosen": -0.6751303672790527,
      "logits/rejected": -0.7156798243522644,
      "logps/chosen": -74.23109436035156,
      "logps/rejected": -77.84270477294922,
      "loss": 14.0507,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0388188436627388,
      "rewards/margins": -0.01138681173324585,
      "rewards/rejected": 0.05020565912127495,
      "step": 444
    },
    {
      "epoch": 0.2584654701748272,
      "grad_norm": 296.38922119140625,
      "learning_rate": 4.353573503776874e-06,
      "logits/chosen": -0.8190110921859741,
      "logits/rejected": -0.6782156229019165,
      "logps/chosen": -73.74525451660156,
      "logps/rejected": -72.44633483886719,
      "loss": 13.3661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07304216921329498,
      "rewards/margins": 0.05356328561902046,
      "rewards/rejected": 0.019478892907500267,
      "step": 445
    },
    {
      "epoch": 0.25904629145611896,
      "grad_norm": 310.6565246582031,
      "learning_rate": 4.352120859965137e-06,
      "logits/chosen": -0.5905870795249939,
      "logits/rejected": -0.6286332011222839,
      "logps/chosen": -73.00715637207031,
      "logps/rejected": -72.94271850585938,
      "loss": 13.9914,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0388491153717041,
      "rewards/margins": -0.007616178598254919,
      "rewards/rejected": 0.04646529257297516,
      "step": 446
    },
    {
      "epoch": 0.2596271127374107,
      "grad_norm": 352.046875,
      "learning_rate": 4.3506682161534e-06,
      "logits/chosen": -0.7802606225013733,
      "logits/rejected": -0.7054556608200073,
      "logps/chosen": -85.91800689697266,
      "logps/rejected": -72.05784606933594,
      "loss": 14.0159,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.019779253751039505,
      "rewards/margins": -0.008547335863113403,
      "rewards/rejected": 0.02832658588886261,
      "step": 447
    },
    {
      "epoch": 0.26020793401870246,
      "grad_norm": 362.30511474609375,
      "learning_rate": 4.3492155723416626e-06,
      "logits/chosen": -0.6785917282104492,
      "logits/rejected": -0.6494299173355103,
      "logps/chosen": -78.44164276123047,
      "logps/rejected": -72.85713958740234,
      "loss": 14.2555,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010164814069867134,
      "rewards/margins": -0.03138233348727226,
      "rewards/rejected": 0.04154714569449425,
      "step": 448
    },
    {
      "epoch": 0.2607887552999942,
      "grad_norm": 317.1739196777344,
      "learning_rate": 4.347762928529925e-06,
      "logits/chosen": -0.6515554189682007,
      "logits/rejected": -0.6316680908203125,
      "logps/chosen": -70.87915802001953,
      "logps/rejected": -73.50413513183594,
      "loss": 13.6688,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02080373838543892,
      "rewards/margins": 0.025081777945160866,
      "rewards/rejected": -0.0042780423536896706,
      "step": 449
    },
    {
      "epoch": 0.26136957658128596,
      "grad_norm": 315.79498291015625,
      "learning_rate": 4.346310284718187e-06,
      "logits/chosen": -0.6134510040283203,
      "logits/rejected": -0.5557273626327515,
      "logps/chosen": -74.24983215332031,
      "logps/rejected": -73.60564422607422,
      "loss": 13.8965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027813846245408058,
      "rewards/margins": 0.00493066618219018,
      "rewards/rejected": 0.022883176803588867,
      "step": 450
    },
    {
      "epoch": 0.2619503978625777,
      "grad_norm": 313.5321350097656,
      "learning_rate": 4.34485764090645e-06,
      "logits/chosen": -0.6806867718696594,
      "logits/rejected": -0.6897571086883545,
      "logps/chosen": -70.46429443359375,
      "logps/rejected": -72.20818328857422,
      "loss": 14.0127,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0054542249999940395,
      "rewards/margins": -0.009584503248333931,
      "rewards/rejected": 0.015038728713989258,
      "step": 451
    },
    {
      "epoch": 0.26253121914386945,
      "grad_norm": 314.8046875,
      "learning_rate": 4.343404997094713e-06,
      "logits/chosen": -0.5489099621772766,
      "logits/rejected": -0.6308473348617554,
      "logps/chosen": -73.40983581542969,
      "logps/rejected": -69.65528106689453,
      "loss": 13.9858,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.021905479952692986,
      "rewards/margins": -0.0018030557548627257,
      "rewards/rejected": 0.023708533495664597,
      "step": 452
    },
    {
      "epoch": 0.2631120404251612,
      "grad_norm": 567.4598388671875,
      "learning_rate": 4.3419523532829754e-06,
      "logits/chosen": -0.6974093914031982,
      "logits/rejected": -0.7631121873855591,
      "logps/chosen": -70.07582092285156,
      "logps/rejected": -81.98683166503906,
      "loss": 13.3815,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05924884229898453,
      "rewards/margins": 0.05474211648106575,
      "rewards/rejected": 0.004506723489612341,
      "step": 453
    },
    {
      "epoch": 0.2636928617064529,
      "grad_norm": 343.7535400390625,
      "learning_rate": 4.340499709471238e-06,
      "logits/chosen": -0.6991898417472839,
      "logits/rejected": -0.6066843867301941,
      "logps/chosen": -75.21280670166016,
      "logps/rejected": -74.64125061035156,
      "loss": 14.3191,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.0069407084956765175,
      "rewards/margins": -0.04137198626995087,
      "rewards/rejected": 0.04831269383430481,
      "step": 454
    },
    {
      "epoch": 0.2642736829877447,
      "grad_norm": 317.2054138183594,
      "learning_rate": 4.339047065659501e-06,
      "logits/chosen": -0.7418617010116577,
      "logits/rejected": -0.7775954008102417,
      "logps/chosen": -69.52796936035156,
      "logps/rejected": -70.95177459716797,
      "loss": 13.4347,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04331202059984207,
      "rewards/margins": 0.04723655804991722,
      "rewards/rejected": -0.00392454257234931,
      "step": 455
    },
    {
      "epoch": 0.2648545042690364,
      "grad_norm": 315.6234130859375,
      "learning_rate": 4.337594421847764e-06,
      "logits/chosen": -0.7921939492225647,
      "logits/rejected": -0.9472505450248718,
      "logps/chosen": -69.9919662475586,
      "logps/rejected": -82.42518615722656,
      "loss": 13.9178,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.0388253815472126,
      "rewards/margins": 0.01137163583189249,
      "rewards/rejected": 0.027453750371932983,
      "step": 456
    },
    {
      "epoch": 0.26543532555032817,
      "grad_norm": 308.3517150878906,
      "learning_rate": 4.336141778036026e-06,
      "logits/chosen": -0.6653806567192078,
      "logits/rejected": -0.5663945078849792,
      "logps/chosen": -73.91642761230469,
      "logps/rejected": -70.58480834960938,
      "loss": 13.8471,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.057994864881038666,
      "rewards/margins": 0.004966372158378363,
      "rewards/rejected": 0.05302848666906357,
      "step": 457
    },
    {
      "epoch": 0.2660161468316199,
      "grad_norm": 325.29302978515625,
      "learning_rate": 4.334689134224288e-06,
      "logits/chosen": -0.5894922018051147,
      "logits/rejected": -0.6214415431022644,
      "logps/chosen": -74.24130249023438,
      "logps/rejected": -80.94917297363281,
      "loss": 14.5074,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.005856601521372795,
      "rewards/margins": -0.0573112778365612,
      "rewards/rejected": 0.06316788494586945,
      "step": 458
    },
    {
      "epoch": 0.26659696811291167,
      "grad_norm": 283.90478515625,
      "learning_rate": 4.333236490412551e-06,
      "logits/chosen": -0.657735288143158,
      "logits/rejected": -0.6617435216903687,
      "logps/chosen": -76.9471435546875,
      "logps/rejected": -68.36034393310547,
      "loss": 13.1023,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.07249955832958221,
      "rewards/margins": 0.08381040394306183,
      "rewards/rejected": -0.011310835368931293,
      "step": 459
    },
    {
      "epoch": 0.2671777893942034,
      "grad_norm": 302.0826416015625,
      "learning_rate": 4.331783846600814e-06,
      "logits/chosen": -0.7745707035064697,
      "logits/rejected": -0.6950326561927795,
      "logps/chosen": -69.80528259277344,
      "logps/rejected": -73.91972351074219,
      "loss": 13.9191,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.027396023273468018,
      "rewards/margins": -0.00236126477830112,
      "rewards/rejected": 0.029757294803857803,
      "step": 460
    },
    {
      "epoch": 0.26775861067549517,
      "grad_norm": 312.1777648925781,
      "learning_rate": 4.330331202789077e-06,
      "logits/chosen": -0.6571930050849915,
      "logits/rejected": -0.7029620409011841,
      "logps/chosen": -78.16141510009766,
      "logps/rejected": -80.50270080566406,
      "loss": 13.3549,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04237108677625656,
      "rewards/margins": 0.05985882878303528,
      "rewards/rejected": -0.01748773828148842,
      "step": 461
    },
    {
      "epoch": 0.2683394319567869,
      "grad_norm": 314.3011169433594,
      "learning_rate": 4.328878558977339e-06,
      "logits/chosen": -0.6543633937835693,
      "logits/rejected": -0.6337400674819946,
      "logps/chosen": -67.81694030761719,
      "logps/rejected": -76.94734954833984,
      "loss": 13.7697,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04975013807415962,
      "rewards/margins": 0.016220813617110252,
      "rewards/rejected": 0.03352931886911392,
      "step": 462
    },
    {
      "epoch": 0.26892025323807867,
      "grad_norm": 287.1407775878906,
      "learning_rate": 4.327425915165602e-06,
      "logits/chosen": -0.7276408076286316,
      "logits/rejected": -0.7486574649810791,
      "logps/chosen": -69.2403335571289,
      "logps/rejected": -60.5196533203125,
      "loss": 13.1441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06597266346216202,
      "rewards/margins": 0.07840771973133087,
      "rewards/rejected": -0.01243506371974945,
      "step": 463
    },
    {
      "epoch": 0.2695010745193704,
      "grad_norm": 307.4203796386719,
      "learning_rate": 4.325973271353864e-06,
      "logits/chosen": -0.7927466630935669,
      "logits/rejected": -0.8723392486572266,
      "logps/chosen": -71.81956481933594,
      "logps/rejected": -71.75994110107422,
      "loss": 13.1263,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05850011110305786,
      "rewards/margins": 0.08116643130779266,
      "rewards/rejected": -0.02266632579267025,
      "step": 464
    },
    {
      "epoch": 0.27008189580066216,
      "grad_norm": 299.2955017089844,
      "learning_rate": 4.324520627542127e-06,
      "logits/chosen": -0.6696484088897705,
      "logits/rejected": -0.6967477798461914,
      "logps/chosen": -64.9903564453125,
      "logps/rejected": -69.08036041259766,
      "loss": 13.864,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.025707131251692772,
      "rewards/margins": 0.0107659213244915,
      "rewards/rejected": 0.014941206201910973,
      "step": 465
    },
    {
      "epoch": 0.2706627170819539,
      "grad_norm": 328.59283447265625,
      "learning_rate": 4.3230679837303895e-06,
      "logits/chosen": -0.8180822134017944,
      "logits/rejected": -0.7941353917121887,
      "logps/chosen": -77.0251693725586,
      "logps/rejected": -79.32392883300781,
      "loss": 12.6981,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.08409181982278824,
      "rewards/margins": 0.12518665194511414,
      "rewards/rejected": -0.0410948283970356,
      "step": 466
    },
    {
      "epoch": 0.2712435383632456,
      "grad_norm": 302.0883483886719,
      "learning_rate": 4.321615339918652e-06,
      "logits/chosen": -0.6744131445884705,
      "logits/rejected": -0.7126671671867371,
      "logps/chosen": -82.62992858886719,
      "logps/rejected": -68.52928161621094,
      "loss": 13.8315,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04055127874016762,
      "rewards/margins": 0.00927782617509365,
      "rewards/rejected": 0.03127345070242882,
      "step": 467
    },
    {
      "epoch": 0.2718243596445374,
      "grad_norm": 296.6047668457031,
      "learning_rate": 4.320162696106915e-06,
      "logits/chosen": -0.6205381155014038,
      "logits/rejected": -0.6734114289283752,
      "logps/chosen": -72.38980865478516,
      "logps/rejected": -68.17768859863281,
      "loss": 13.3424,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.059192102402448654,
      "rewards/margins": 0.06497955322265625,
      "rewards/rejected": -0.005787448026239872,
      "step": 468
    },
    {
      "epoch": 0.2724051809258291,
      "grad_norm": 293.75018310546875,
      "learning_rate": 4.318710052295178e-06,
      "logits/chosen": -0.8345297574996948,
      "logits/rejected": -0.8630663752555847,
      "logps/chosen": -66.68868255615234,
      "logps/rejected": -69.19453430175781,
      "loss": 13.0063,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.06119300797581673,
      "rewards/margins": 0.09242288768291473,
      "rewards/rejected": -0.031229889020323753,
      "step": 469
    },
    {
      "epoch": 0.2729860022071209,
      "grad_norm": 330.23846435546875,
      "learning_rate": 4.3172574084834405e-06,
      "logits/chosen": -0.6324206590652466,
      "logits/rejected": -0.6938502192497253,
      "logps/chosen": -74.11239624023438,
      "logps/rejected": -73.95758056640625,
      "loss": 14.0991,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0044637094251811504,
      "rewards/margins": -0.018323037773370743,
      "rewards/rejected": 0.01385932881385088,
      "step": 470
    },
    {
      "epoch": 0.2735668234884126,
      "grad_norm": 291.638671875,
      "learning_rate": 4.315804764671702e-06,
      "logits/chosen": -0.6697665452957153,
      "logits/rejected": -0.6289907693862915,
      "logps/chosen": -72.07844543457031,
      "logps/rejected": -68.84980773925781,
      "loss": 13.8797,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03210974857211113,
      "rewards/margins": 0.0021261770743876696,
      "rewards/rejected": 0.02998356893658638,
      "step": 471
    },
    {
      "epoch": 0.2741476447697044,
      "grad_norm": 326.5478515625,
      "learning_rate": 4.314352120859965e-06,
      "logits/chosen": -0.6647511124610901,
      "logits/rejected": -0.6528132557868958,
      "logps/chosen": -72.49028778076172,
      "logps/rejected": -72.65093231201172,
      "loss": 14.1427,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.015104440040886402,
      "rewards/margins": -0.022749561816453934,
      "rewards/rejected": 0.037853993475437164,
      "step": 472
    },
    {
      "epoch": 0.2747284660509961,
      "grad_norm": 330.836181640625,
      "learning_rate": 4.312899477048228e-06,
      "logits/chosen": -0.5921165347099304,
      "logits/rejected": -0.6261785626411438,
      "logps/chosen": -83.95507049560547,
      "logps/rejected": -69.69046020507812,
      "loss": 13.5342,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.049983587116003036,
      "rewards/margins": 0.041167039424180984,
      "rewards/rejected": 0.008816548623144627,
      "step": 473
    },
    {
      "epoch": 0.2753092873322879,
      "grad_norm": 286.28839111328125,
      "learning_rate": 4.311446833236491e-06,
      "logits/chosen": -0.5407285690307617,
      "logits/rejected": -0.5518749356269836,
      "logps/chosen": -69.38809967041016,
      "logps/rejected": -64.3993911743164,
      "loss": 13.7355,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03605775907635689,
      "rewards/margins": 0.015156927518546581,
      "rewards/rejected": 0.02090083435177803,
      "step": 474
    },
    {
      "epoch": 0.2758901086135796,
      "grad_norm": 321.68450927734375,
      "learning_rate": 4.309994189424753e-06,
      "logits/chosen": -0.7276217341423035,
      "logits/rejected": -0.8036016225814819,
      "logps/chosen": -70.2737045288086,
      "logps/rejected": -66.9681625366211,
      "loss": 13.9062,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0001468472182750702,
      "rewards/margins": -0.00017342269711662084,
      "rewards/rejected": 0.00032026879489421844,
      "step": 475
    },
    {
      "epoch": 0.2764709298948714,
      "grad_norm": 306.3502502441406,
      "learning_rate": 4.308541545613016e-06,
      "logits/chosen": -0.7900197505950928,
      "logits/rejected": -0.6438810229301453,
      "logps/chosen": -69.17347717285156,
      "logps/rejected": -62.60564422607422,
      "loss": 13.6038,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04666885733604431,
      "rewards/margins": 0.03157535940408707,
      "rewards/rejected": 0.015093490481376648,
      "step": 476
    },
    {
      "epoch": 0.2770517511761631,
      "grad_norm": 382.4345397949219,
      "learning_rate": 4.307088901801279e-06,
      "logits/chosen": -0.8302785158157349,
      "logits/rejected": -0.830335795879364,
      "logps/chosen": -83.38411712646484,
      "logps/rejected": -72.17436981201172,
      "loss": 13.7722,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02929898537695408,
      "rewards/margins": 0.015593672171235085,
      "rewards/rejected": 0.013705313205718994,
      "step": 477
    },
    {
      "epoch": 0.2776325724574548,
      "grad_norm": 292.42779541015625,
      "learning_rate": 4.305636257989541e-06,
      "logits/chosen": -0.6505283713340759,
      "logits/rejected": -0.5827508568763733,
      "logps/chosen": -64.67672729492188,
      "logps/rejected": -68.16065979003906,
      "loss": 13.2237,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05520979315042496,
      "rewards/margins": 0.0683043897151947,
      "rewards/rejected": -0.013094606809318066,
      "step": 478
    },
    {
      "epoch": 0.2782133937387466,
      "grad_norm": 321.1835632324219,
      "learning_rate": 4.3041836141778035e-06,
      "logits/chosen": -0.4862455725669861,
      "logits/rejected": -0.4693034589290619,
      "logps/chosen": -70.4814224243164,
      "logps/rejected": -84.19317626953125,
      "loss": 13.6491,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.016184937208890915,
      "rewards/margins": 0.03135867789387703,
      "rewards/rejected": -0.015173738822340965,
      "step": 479
    },
    {
      "epoch": 0.2787942150200383,
      "grad_norm": 311.97564697265625,
      "learning_rate": 4.302730970366066e-06,
      "logits/chosen": -0.554233193397522,
      "logits/rejected": -0.5739291310310364,
      "logps/chosen": -71.45389556884766,
      "logps/rejected": -76.47578430175781,
      "loss": 13.6573,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.028831113129854202,
      "rewards/margins": 0.03090267814695835,
      "rewards/rejected": -0.0020715640857815742,
      "step": 480
    },
    {
      "epoch": 0.2793750363013301,
      "grad_norm": 309.7761535644531,
      "learning_rate": 4.301278326554329e-06,
      "logits/chosen": -0.6836757659912109,
      "logits/rejected": -0.7563328742980957,
      "logps/chosen": -65.9247817993164,
      "logps/rejected": -76.11883544921875,
      "loss": 14.0186,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06773559749126434,
      "rewards/margins": -0.002912606345489621,
      "rewards/rejected": 0.0706482082605362,
      "step": 481
    },
    {
      "epoch": 0.2799558575826218,
      "grad_norm": 301.5060729980469,
      "learning_rate": 4.299825682742592e-06,
      "logits/chosen": -0.7972155809402466,
      "logits/rejected": -0.7350047826766968,
      "logps/chosen": -67.74440002441406,
      "logps/rejected": -71.73310089111328,
      "loss": 13.6848,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.025806616991758347,
      "rewards/margins": 0.022161057218909264,
      "rewards/rejected": 0.0036455602385103703,
      "step": 482
    },
    {
      "epoch": 0.2805366788639136,
      "grad_norm": 301.0857849121094,
      "learning_rate": 4.2983730389308545e-06,
      "logits/chosen": -0.6307353973388672,
      "logits/rejected": -0.5791727900505066,
      "logps/chosen": -70.24183654785156,
      "logps/rejected": -69.36378479003906,
      "loss": 13.4857,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.06295748054981232,
      "rewards/margins": 0.043685734272003174,
      "rewards/rejected": 0.019271746277809143,
      "step": 483
    },
    {
      "epoch": 0.2811175001452053,
      "grad_norm": 685.1707763671875,
      "learning_rate": 4.296920395119117e-06,
      "logits/chosen": -0.7495703101158142,
      "logits/rejected": -0.678167998790741,
      "logps/chosen": -77.27181243896484,
      "logps/rejected": -72.3003158569336,
      "loss": 13.8755,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.010486298240721226,
      "rewards/margins": 0.006247940473258495,
      "rewards/rejected": 0.004238357301801443,
      "step": 484
    },
    {
      "epoch": 0.2816983214264971,
      "grad_norm": 294.2120056152344,
      "learning_rate": 4.29546775130738e-06,
      "logits/chosen": -0.5442952513694763,
      "logits/rejected": -0.6088670492172241,
      "logps/chosen": -73.0514907836914,
      "logps/rejected": -73.78404235839844,
      "loss": 13.6341,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0040179165080189705,
      "rewards/margins": 0.027353759855031967,
      "rewards/rejected": -0.03137167543172836,
      "step": 485
    },
    {
      "epoch": 0.2822791427077888,
      "grad_norm": 306.35723876953125,
      "learning_rate": 4.294015107495643e-06,
      "logits/chosen": -0.6691688299179077,
      "logits/rejected": -0.7110737562179565,
      "logps/chosen": -68.50287628173828,
      "logps/rejected": -65.523193359375,
      "loss": 13.5978,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.014910449273884296,
      "rewards/margins": 0.03183292597532272,
      "rewards/rejected": -0.016922477632761,
      "step": 486
    },
    {
      "epoch": 0.2828599639890806,
      "grad_norm": 314.25897216796875,
      "learning_rate": 4.2925624636839055e-06,
      "logits/chosen": -0.6867062449455261,
      "logits/rejected": -0.6559956073760986,
      "logps/chosen": -86.4985122680664,
      "logps/rejected": -67.22306060791016,
      "loss": 13.7589,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01645682193338871,
      "rewards/margins": 0.017809275537729263,
      "rewards/rejected": -0.001352452440187335,
      "step": 487
    },
    {
      "epoch": 0.2834407852703723,
      "grad_norm": 288.4145812988281,
      "learning_rate": 4.291109819872168e-06,
      "logits/chosen": -0.6363990902900696,
      "logits/rejected": -0.6217866539955139,
      "logps/chosen": -72.58358764648438,
      "logps/rejected": -71.9157485961914,
      "loss": 13.0157,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0661824643611908,
      "rewards/margins": 0.09238552302122116,
      "rewards/rejected": -0.026203066110610962,
      "step": 488
    },
    {
      "epoch": 0.28402160655166403,
      "grad_norm": 304.28765869140625,
      "learning_rate": 4.28965717606043e-06,
      "logits/chosen": -0.7047585248947144,
      "logits/rejected": -0.6908853054046631,
      "logps/chosen": -69.95263671875,
      "logps/rejected": -71.68666076660156,
      "loss": 13.3886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04382898285984993,
      "rewards/margins": 0.059452660381793976,
      "rewards/rejected": -0.015623673796653748,
      "step": 489
    },
    {
      "epoch": 0.2846024278329558,
      "grad_norm": 314.20196533203125,
      "learning_rate": 4.288204532248693e-06,
      "logits/chosen": -0.7270024418830872,
      "logits/rejected": -0.7925176620483398,
      "logps/chosen": -73.37562561035156,
      "logps/rejected": -85.60382843017578,
      "loss": 13.1235,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03009481355547905,
      "rewards/margins": 0.0827663242816925,
      "rewards/rejected": -0.052671510726213455,
      "step": 490
    },
    {
      "epoch": 0.28518324911424753,
      "grad_norm": 296.90582275390625,
      "learning_rate": 4.286751888436956e-06,
      "logits/chosen": -0.6854633092880249,
      "logits/rejected": -0.7035878896713257,
      "logps/chosen": -68.50250244140625,
      "logps/rejected": -67.49859619140625,
      "loss": 13.7267,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.030942970886826515,
      "rewards/margins": 0.019369639456272125,
      "rewards/rejected": 0.011573335155844688,
      "step": 491
    },
    {
      "epoch": 0.2857640703955393,
      "grad_norm": 292.898681640625,
      "learning_rate": 4.285299244625218e-06,
      "logits/chosen": -0.5533262491226196,
      "logits/rejected": -0.6104485392570496,
      "logps/chosen": -72.7857666015625,
      "logps/rejected": -67.43394470214844,
      "loss": 13.9817,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.005316922441124916,
      "rewards/margins": -0.0064797671511769295,
      "rewards/rejected": 0.0011628434294834733,
      "step": 492
    },
    {
      "epoch": 0.286344891676831,
      "grad_norm": 323.1135559082031,
      "learning_rate": 4.283846600813481e-06,
      "logits/chosen": -0.5882741808891296,
      "logits/rejected": -0.6077271699905396,
      "logps/chosen": -78.02590942382812,
      "logps/rejected": -83.53060150146484,
      "loss": 13.4642,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007745922543108463,
      "rewards/margins": 0.050111640244722366,
      "rewards/rejected": -0.04236571118235588,
      "step": 493
    },
    {
      "epoch": 0.2869257129581228,
      "grad_norm": 377.0531921386719,
      "learning_rate": 4.282393957001744e-06,
      "logits/chosen": -0.659203827381134,
      "logits/rejected": -0.6870671510696411,
      "logps/chosen": -68.08976745605469,
      "logps/rejected": -78.85200500488281,
      "loss": 13.4473,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.050796620547771454,
      "rewards/margins": 0.04922497272491455,
      "rewards/rejected": 0.0015716440975666046,
      "step": 494
    },
    {
      "epoch": 0.2875065342394145,
      "grad_norm": 575.7816162109375,
      "learning_rate": 4.280941313190007e-06,
      "logits/chosen": -0.6494874954223633,
      "logits/rejected": -0.6947387456893921,
      "logps/chosen": -73.44721984863281,
      "logps/rejected": -77.6927719116211,
      "loss": 14.4423,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.00039952099905349314,
      "rewards/margins": -0.050260018557310104,
      "rewards/rejected": 0.05065953731536865,
      "step": 495
    },
    {
      "epoch": 0.2880873555207063,
      "grad_norm": 292.5953369140625,
      "learning_rate": 4.2794886693782685e-06,
      "logits/chosen": -0.745873749256134,
      "logits/rejected": -0.6551159024238586,
      "logps/chosen": -68.93890380859375,
      "logps/rejected": -70.55396270751953,
      "loss": 13.5985,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01787867583334446,
      "rewards/margins": 0.032074641436338425,
      "rewards/rejected": -0.01419596653431654,
      "step": 496
    },
    {
      "epoch": 0.288668176801998,
      "grad_norm": 317.0806884765625,
      "learning_rate": 4.278036025566531e-06,
      "logits/chosen": -0.6900259256362915,
      "logits/rejected": -0.6669089198112488,
      "logps/chosen": -72.48905944824219,
      "logps/rejected": -75.67109680175781,
      "loss": 13.8434,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.016828546300530434,
      "rewards/margins": 0.006076467223465443,
      "rewards/rejected": 0.010752077214419842,
      "step": 497
    },
    {
      "epoch": 0.2892489980832898,
      "grad_norm": 450.179931640625,
      "learning_rate": 4.276583381754794e-06,
      "logits/chosen": -0.7485173344612122,
      "logits/rejected": -0.8365989923477173,
      "logps/chosen": -72.33958435058594,
      "logps/rejected": -69.51062774658203,
      "loss": 13.3385,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0609331838786602,
      "rewards/margins": 0.060974687337875366,
      "rewards/rejected": -4.150420500081964e-05,
      "step": 498
    },
    {
      "epoch": 0.2898298193645815,
      "grad_norm": 319.4369201660156,
      "learning_rate": 4.275130737943057e-06,
      "logits/chosen": -0.659496545791626,
      "logits/rejected": -0.6515854001045227,
      "logps/chosen": -70.12030029296875,
      "logps/rejected": -75.98362731933594,
      "loss": 13.3334,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.026825930923223495,
      "rewards/margins": 0.0615021288394928,
      "rewards/rejected": -0.03467618674039841,
      "step": 499
    },
    {
      "epoch": 0.29041064064587324,
      "grad_norm": 317.2661437988281,
      "learning_rate": 4.2736780941313195e-06,
      "logits/chosen": -0.6833754777908325,
      "logits/rejected": -0.6367133855819702,
      "logps/chosen": -81.86927795410156,
      "logps/rejected": -78.07447814941406,
      "loss": 13.1256,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07352960854768753,
      "rewards/margins": 0.08241195976734161,
      "rewards/rejected": -0.008882349357008934,
      "step": 500
    }
  ],
  "logging_steps": 1,
  "max_steps": 3442,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}