{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9992254066615027,
  "eval_steps": 100,
  "global_step": 2904,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.7182130584192438e-09,
      "logits/chosen": -2.293962240219116,
      "logits/rejected": -2.235898017883301,
      "logps/chosen": -280.7555847167969,
      "logps/rejected": -204.8462371826172,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 1.718213058419244e-08,
      "logits/chosen": -2.4112908840179443,
      "logits/rejected": -2.3392038345336914,
      "logps/chosen": -294.24664306640625,
      "logps/rejected": -213.90679931640625,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002664534840732813,
      "rewards/margins": -9.001558646559715e-05,
      "rewards/rejected": 0.00275455042719841,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 3.436426116838488e-08,
      "logits/chosen": -2.4150779247283936,
      "logits/rejected": -2.3804850578308105,
      "logps/chosen": -279.4464416503906,
      "logps/rejected": -237.6862335205078,
      "loss": 0.6923,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.003086227923631668,
      "rewards/margins": 0.009707429446280003,
      "rewards/rejected": -0.006621202919632196,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 5.154639175257731e-08,
      "logits/chosen": -2.4616353511810303,
      "logits/rejected": -2.3937487602233887,
      "logps/chosen": -301.0641174316406,
      "logps/rejected": -215.75418090820312,
      "loss": 0.6935,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.0004951163427904248,
      "rewards/margins": 0.000455385452369228,
      "rewards/rejected": 3.9730781281832606e-05,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 6.872852233676976e-08,
      "logits/chosen": -2.3844287395477295,
      "logits/rejected": -2.344656467437744,
      "logps/chosen": -291.42413330078125,
      "logps/rejected": -231.827880859375,
      "loss": 0.6935,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0011853076284751296,
      "rewards/margins": 0.002922216197475791,
      "rewards/rejected": -0.0017369084525853395,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 8.59106529209622e-08,
      "logits/chosen": -2.4430339336395264,
      "logits/rejected": -2.382826566696167,
      "logps/chosen": -299.23663330078125,
      "logps/rejected": -220.186767578125,
      "loss": 0.6932,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0030969330109655857,
      "rewards/margins": 0.004286954645067453,
      "rewards/rejected": -0.0011900208191946149,
      "step": 50
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.0309278350515462e-07,
      "logits/chosen": -2.4357097148895264,
      "logits/rejected": -2.4250669479370117,
      "logps/chosen": -272.5534973144531,
      "logps/rejected": -227.50332641601562,
      "loss": 0.6936,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.00050017872126773,
      "rewards/margins": 0.0015349002787843347,
      "rewards/rejected": -0.0020350790582597256,
      "step": 60
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.202749140893471e-07,
      "logits/chosen": -2.465843677520752,
      "logits/rejected": -2.4208121299743652,
      "logps/chosen": -292.08990478515625,
      "logps/rejected": -206.94558715820312,
      "loss": 0.6919,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.0012726224958896637,
      "rewards/margins": 0.003322723787277937,
      "rewards/rejected": -0.0020501010585576296,
      "step": 70
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.3745704467353952e-07,
      "logits/chosen": -2.3812198638916016,
      "logits/rejected": -2.390000104904175,
      "logps/chosen": -250.2107391357422,
      "logps/rejected": -212.4706268310547,
      "loss": 0.6901,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004158427938818932,
      "rewards/margins": 0.005836354102939367,
      "rewards/rejected": -0.0016779262805357575,
      "step": 80
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.5463917525773197e-07,
      "logits/chosen": -2.3327183723449707,
      "logits/rejected": -2.254629611968994,
      "logps/chosen": -241.6092529296875,
      "logps/rejected": -185.40869140625,
      "loss": 0.6951,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.005251345224678516,
      "rewards/margins": -0.004248884506523609,
      "rewards/rejected": -0.0010024613002315164,
      "step": 90
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.718213058419244e-07,
      "logits/chosen": -2.401676654815674,
      "logits/rejected": -2.3915371894836426,
      "logps/chosen": -259.74755859375,
      "logps/rejected": -221.87295532226562,
      "loss": 0.6923,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.005552566610276699,
      "rewards/margins": -0.0006124695646576583,
      "rewards/rejected": 0.006165036000311375,
      "step": 100
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.8900343642611682e-07,
      "logits/chosen": -2.441145181655884,
      "logits/rejected": -2.311302423477173,
      "logps/chosen": -252.06753540039062,
      "logps/rejected": -210.41696166992188,
      "loss": 0.6889,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.0075301057659089565,
      "rewards/margins": 0.013402941636741161,
      "rewards/rejected": -0.0058728354051709175,
      "step": 110
    },
    {
      "epoch": 0.12,
      "learning_rate": 2.0618556701030925e-07,
      "logits/chosen": -2.3808252811431885,
      "logits/rejected": -2.4377057552337646,
      "logps/chosen": -258.4937438964844,
      "logps/rejected": -210.9638671875,
      "loss": 0.6905,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0047661615535616875,
      "rewards/margins": 0.008594167418777943,
      "rewards/rejected": -0.003828004002571106,
      "step": 120
    },
    {
      "epoch": 0.13,
      "learning_rate": 2.2336769759450173e-07,
      "logits/chosen": -2.447084903717041,
      "logits/rejected": -2.4316954612731934,
      "logps/chosen": -279.2828674316406,
      "logps/rejected": -213.9290313720703,
      "loss": 0.6873,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.015633000060915947,
      "rewards/margins": 0.019236568361520767,
      "rewards/rejected": -0.0036035701632499695,
      "step": 130
    },
    {
      "epoch": 0.14,
      "learning_rate": 2.405498281786942e-07,
      "logits/chosen": -2.574856996536255,
      "logits/rejected": -2.4002137184143066,
      "logps/chosen": -267.64520263671875,
      "logps/rejected": -203.2295684814453,
      "loss": 0.6892,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.008941258303821087,
      "rewards/margins": 0.014786717481911182,
      "rewards/rejected": -0.00584546010941267,
      "step": 140
    },
    {
      "epoch": 0.15,
      "learning_rate": 2.5773195876288655e-07,
      "logits/chosen": -2.3573081493377686,
      "logits/rejected": -2.390941619873047,
      "logps/chosen": -280.3595886230469,
      "logps/rejected": -214.0395965576172,
      "loss": 0.6876,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": 0.0026760019827634096,
      "rewards/margins": 0.010370850563049316,
      "rewards/rejected": -0.0076948488131165504,
      "step": 150
    },
    {
      "epoch": 0.17,
      "learning_rate": 2.7491408934707903e-07,
      "logits/chosen": -2.460977077484131,
      "logits/rejected": -2.4357001781463623,
      "logps/chosen": -307.5310974121094,
      "logps/rejected": -234.86721801757812,
      "loss": 0.6857,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.010203736834228039,
      "rewards/margins": 0.017851073294878006,
      "rewards/rejected": -0.0076473369263112545,
      "step": 160
    },
    {
      "epoch": 0.18,
      "learning_rate": 2.9209621993127146e-07,
      "logits/chosen": -2.4167332649230957,
      "logits/rejected": -2.4196879863739014,
      "logps/chosen": -284.23834228515625,
      "logps/rejected": -220.6202392578125,
      "loss": 0.6861,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.006566309370100498,
      "rewards/margins": 0.00825690571218729,
      "rewards/rejected": -0.001690595643594861,
      "step": 170
    },
    {
      "epoch": 0.19,
      "learning_rate": 3.0927835051546394e-07,
      "logits/chosen": -2.426633358001709,
      "logits/rejected": -2.426823139190674,
      "logps/chosen": -261.1161193847656,
      "logps/rejected": -236.690673828125,
      "loss": 0.682,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.007787441369146109,
      "rewards/margins": 0.021113665774464607,
      "rewards/rejected": -0.013326220214366913,
      "step": 180
    },
    {
      "epoch": 0.2,
      "learning_rate": 3.2646048109965636e-07,
      "logits/chosen": -2.47978138923645,
      "logits/rejected": -2.393406391143799,
      "logps/chosen": -261.36138916015625,
      "logps/rejected": -213.78268432617188,
      "loss": 0.6807,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.010398685932159424,
      "rewards/margins": 0.022776171565055847,
      "rewards/rejected": -0.012377489358186722,
      "step": 190
    },
    {
      "epoch": 0.21,
      "learning_rate": 3.436426116838488e-07,
      "logits/chosen": -2.421168565750122,
      "logits/rejected": -2.3462955951690674,
      "logps/chosen": -262.1727294921875,
      "logps/rejected": -223.8699188232422,
      "loss": 0.6806,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.012244177982211113,
      "rewards/margins": 0.029001509770751,
      "rewards/rejected": -0.016757333651185036,
      "step": 200
    },
    {
      "epoch": 0.22,
      "learning_rate": 3.608247422680412e-07,
      "logits/chosen": -2.446596145629883,
      "logits/rejected": -2.367396593093872,
      "logps/chosen": -276.6649475097656,
      "logps/rejected": -203.23178100585938,
      "loss": 0.6758,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.012773854658007622,
      "rewards/margins": 0.03576343134045601,
      "rewards/rejected": -0.022989574819803238,
      "step": 210
    },
    {
      "epoch": 0.23,
      "learning_rate": 3.7800687285223364e-07,
      "logits/chosen": -2.4427998065948486,
      "logits/rejected": -2.400299549102783,
      "logps/chosen": -248.6843719482422,
      "logps/rejected": -211.16293334960938,
      "loss": 0.674,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.01836214028298855,
      "rewards/margins": 0.0383928082883358,
      "rewards/rejected": -0.020030666142702103,
      "step": 220
    },
    {
      "epoch": 0.24,
      "learning_rate": 3.9518900343642607e-07,
      "logits/chosen": -2.4508414268493652,
      "logits/rejected": -2.444462299346924,
      "logps/chosen": -261.3500061035156,
      "logps/rejected": -210.68807983398438,
      "loss": 0.6755,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.02510516345500946,
      "rewards/margins": 0.04327309876680374,
      "rewards/rejected": -0.018167927861213684,
      "step": 230
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.123711340206185e-07,
      "logits/chosen": -2.3069140911102295,
      "logits/rejected": -2.3430962562561035,
      "logps/chosen": -280.17633056640625,
      "logps/rejected": -234.605224609375,
      "loss": 0.6696,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.024332646280527115,
      "rewards/margins": 0.05363120511174202,
      "rewards/rejected": -0.029298555105924606,
      "step": 240
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.2955326460481097e-07,
      "logits/chosen": -2.408478021621704,
      "logits/rejected": -2.38106369972229,
      "logps/chosen": -267.2620849609375,
      "logps/rejected": -210.53933715820312,
      "loss": 0.669,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.022226443514227867,
      "rewards/margins": 0.06285347044467926,
      "rewards/rejected": -0.040627025067806244,
      "step": 250
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.4673539518900345e-07,
      "logits/chosen": -2.3904354572296143,
      "logits/rejected": -2.4175901412963867,
      "logps/chosen": -261.8872375488281,
      "logps/rejected": -215.2822723388672,
      "loss": 0.6659,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.012217490933835506,
      "rewards/margins": 0.041740067303180695,
      "rewards/rejected": -0.029522571712732315,
      "step": 260
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.639175257731959e-07,
      "logits/chosen": -2.3512496948242188,
      "logits/rejected": -2.374730110168457,
      "logps/chosen": -221.1552276611328,
      "logps/rejected": -214.0994110107422,
      "loss": 0.6617,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.02718324586749077,
      "rewards/margins": 0.07020439207553864,
      "rewards/rejected": -0.04302113875746727,
      "step": 270
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.810996563573884e-07,
      "logits/chosen": -2.4109158515930176,
      "logits/rejected": -2.416260242462158,
      "logps/chosen": -265.5227355957031,
      "logps/rejected": -219.8955841064453,
      "loss": 0.6591,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.02033821865916252,
      "rewards/margins": 0.0716070905327797,
      "rewards/rejected": -0.051268868148326874,
      "step": 280
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.982817869415807e-07,
      "logits/chosen": -2.4649157524108887,
      "logits/rejected": -2.336876153945923,
      "logps/chosen": -299.04083251953125,
      "logps/rejected": -233.513671875,
      "loss": 0.6561,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.04183030128479004,
      "rewards/margins": 0.08989714086055756,
      "rewards/rejected": -0.04806683957576752,
      "step": 290
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.982778415614236e-07,
      "logits/chosen": -2.3887062072753906,
      "logits/rejected": -2.34873628616333,
      "logps/chosen": -236.2488250732422,
      "logps/rejected": -220.21450805664062,
      "loss": 0.6523,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.022816497832536697,
      "rewards/margins": 0.09650030732154846,
      "rewards/rejected": -0.07368380576372147,
      "step": 300
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.963643321852277e-07,
      "logits/chosen": -2.425257444381714,
      "logits/rejected": -2.330592393875122,
      "logps/chosen": -285.37896728515625,
      "logps/rejected": -231.2700653076172,
      "loss": 0.6515,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.010598802007734776,
      "rewards/margins": 0.0920100286602974,
      "rewards/rejected": -0.0814112201333046,
      "step": 310
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.944508228090318e-07,
      "logits/chosen": -2.4376401901245117,
      "logits/rejected": -2.4971015453338623,
      "logps/chosen": -238.9815216064453,
      "logps/rejected": -192.4442138671875,
      "loss": 0.6369,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.04103684425354004,
      "rewards/margins": 0.13544507324695587,
      "rewards/rejected": -0.09440822899341583,
      "step": 320
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.925373134328357e-07,
      "logits/chosen": -2.4623780250549316,
      "logits/rejected": -2.439487934112549,
      "logps/chosen": -252.5996551513672,
      "logps/rejected": -222.22421264648438,
      "loss": 0.6422,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.04304979741573334,
      "rewards/margins": 0.10223046690225601,
      "rewards/rejected": -0.05918065458536148,
      "step": 330
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.906238040566398e-07,
      "logits/chosen": -2.4748172760009766,
      "logits/rejected": -2.356078863143921,
      "logps/chosen": -271.64324951171875,
      "logps/rejected": -231.6174774169922,
      "loss": 0.6425,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.041741594672203064,
      "rewards/margins": 0.15150585770606995,
      "rewards/rejected": -0.10976427793502808,
      "step": 340
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.887102946804438e-07,
      "logits/chosen": -2.5339407920837402,
      "logits/rejected": -2.4247193336486816,
      "logps/chosen": -289.107177734375,
      "logps/rejected": -223.7015838623047,
      "loss": 0.6301,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.05352502316236496,
      "rewards/margins": 0.20187970995903015,
      "rewards/rejected": -0.1483546942472458,
      "step": 350
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.867967853042479e-07,
      "logits/chosen": -2.44026255607605,
      "logits/rejected": -2.4695184230804443,
      "logps/chosen": -272.41827392578125,
      "logps/rejected": -223.8868408203125,
      "loss": 0.6206,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.06177166849374771,
      "rewards/margins": 0.19406349956989288,
      "rewards/rejected": -0.13229182362556458,
      "step": 360
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.84883275928052e-07,
      "logits/chosen": -2.4581284523010254,
      "logits/rejected": -2.422722339630127,
      "logps/chosen": -274.32763671875,
      "logps/rejected": -228.18148803710938,
      "loss": 0.6222,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04478592425584793,
      "rewards/margins": 0.19903826713562012,
      "rewards/rejected": -0.1542523354291916,
      "step": 370
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.82969766551856e-07,
      "logits/chosen": -2.365147829055786,
      "logits/rejected": -2.376405954360962,
      "logps/chosen": -258.0191345214844,
      "logps/rejected": -228.0791015625,
      "loss": 0.6345,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.04648314788937569,
      "rewards/margins": 0.18209731578826904,
      "rewards/rejected": -0.13561417162418365,
      "step": 380
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.810562571756601e-07,
      "logits/chosen": -2.418147325515747,
      "logits/rejected": -2.405946969985962,
      "logps/chosen": -255.6215362548828,
      "logps/rejected": -219.8120880126953,
      "loss": 0.6262,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.01959371007978916,
      "rewards/margins": 0.18485160171985626,
      "rewards/rejected": -0.16525788605213165,
      "step": 390
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.791427477994642e-07,
      "logits/chosen": -2.419095516204834,
      "logits/rejected": -2.4036061763763428,
      "logps/chosen": -306.9060974121094,
      "logps/rejected": -239.7941436767578,
      "loss": 0.6195,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.06084824725985527,
      "rewards/margins": 0.20281776785850525,
      "rewards/rejected": -0.14196953177452087,
      "step": 400
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.772292384232682e-07,
      "logits/chosen": -2.417729377746582,
      "logits/rejected": -2.3787436485290527,
      "logps/chosen": -246.24478149414062,
      "logps/rejected": -245.55166625976562,
      "loss": 0.6208,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.024662364274263382,
      "rewards/margins": 0.1620568484067917,
      "rewards/rejected": -0.18671919405460358,
      "step": 410
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.753157290470723e-07,
      "logits/chosen": -2.4877963066101074,
      "logits/rejected": -2.4664740562438965,
      "logps/chosen": -256.0787048339844,
      "logps/rejected": -225.0019989013672,
      "loss": 0.6281,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006953413598239422,
      "rewards/margins": 0.161538228392601,
      "rewards/rejected": -0.15458481013774872,
      "step": 420
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.7340221967087635e-07,
      "logits/chosen": -2.412123918533325,
      "logits/rejected": -2.3679380416870117,
      "logps/chosen": -251.4219970703125,
      "logps/rejected": -224.938720703125,
      "loss": 0.6154,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.0361722894012928,
      "rewards/margins": 0.22137188911437988,
      "rewards/rejected": -0.1851995885372162,
      "step": 430
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.714887102946804e-07,
      "logits/chosen": -2.470468521118164,
      "logits/rejected": -2.3362622261047363,
      "logps/chosen": -257.301513671875,
      "logps/rejected": -200.90008544921875,
      "loss": 0.6021,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.01734147220849991,
      "rewards/margins": 0.22712071239948273,
      "rewards/rejected": -0.20977921783924103,
      "step": 440
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.6957520091848447e-07,
      "logits/chosen": -2.4263899326324463,
      "logits/rejected": -2.3918890953063965,
      "logps/chosen": -228.27206420898438,
      "logps/rejected": -204.10134887695312,
      "loss": 0.6338,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.007703957613557577,
      "rewards/margins": 0.17124322056770325,
      "rewards/rejected": -0.16353924572467804,
      "step": 450
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.6766169154228853e-07,
      "logits/chosen": -2.3560242652893066,
      "logits/rejected": -2.3139007091522217,
      "logps/chosen": -268.39398193359375,
      "logps/rejected": -239.73159790039062,
      "loss": 0.5952,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.06542753428220749,
      "rewards/margins": 0.2891160547733307,
      "rewards/rejected": -0.2236885279417038,
      "step": 460
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.657481821660926e-07,
      "logits/chosen": -2.452782154083252,
      "logits/rejected": -2.4332146644592285,
      "logps/chosen": -283.4723205566406,
      "logps/rejected": -211.5760498046875,
      "loss": 0.5978,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.004657271318137646,
      "rewards/margins": 0.2682662606239319,
      "rewards/rejected": -0.2729235589504242,
      "step": 470
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.6383467278989666e-07,
      "logits/chosen": -2.3910794258117676,
      "logits/rejected": -2.4533417224884033,
      "logps/chosen": -248.0291748046875,
      "logps/rejected": -230.6380157470703,
      "loss": 0.6143,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.015768401324748993,
      "rewards/margins": 0.2060297429561615,
      "rewards/rejected": -0.22179818153381348,
      "step": 480
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.6192116341370067e-07,
      "logits/chosen": -2.5222439765930176,
      "logits/rejected": -2.4398193359375,
      "logps/chosen": -277.2117614746094,
      "logps/rejected": -230.4596710205078,
      "loss": 0.5947,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04367998614907265,
      "rewards/margins": 0.33074626326560974,
      "rewards/rejected": -0.2870662808418274,
      "step": 490
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.6000765403750473e-07,
      "logits/chosen": -2.423351764678955,
      "logits/rejected": -2.3357410430908203,
      "logps/chosen": -255.89846801757812,
      "logps/rejected": -211.130126953125,
      "loss": 0.5989,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.019995370879769325,
      "rewards/margins": 0.26641523838043213,
      "rewards/rejected": -0.24641986191272736,
      "step": 500
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.580941446613088e-07,
      "logits/chosen": -2.4628067016601562,
      "logits/rejected": -2.4185214042663574,
      "logps/chosen": -257.75567626953125,
      "logps/rejected": -238.31454467773438,
      "loss": 0.5993,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.023913182318210602,
      "rewards/margins": 0.23212233185768127,
      "rewards/rejected": -0.2560355067253113,
      "step": 510
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.5618063528511285e-07,
      "logits/chosen": -2.3876211643218994,
      "logits/rejected": -2.357927083969116,
      "logps/chosen": -274.25555419921875,
      "logps/rejected": -232.0738525390625,
      "loss": 0.5993,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.04971958324313164,
      "rewards/margins": 0.2438163459300995,
      "rewards/rejected": -0.29353591799736023,
      "step": 520
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.542671259089169e-07,
      "logits/chosen": -2.4641833305358887,
      "logits/rejected": -2.40731143951416,
      "logps/chosen": -273.7682189941406,
      "logps/rejected": -233.98583984375,
      "loss": 0.6062,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0051215579733252525,
      "rewards/margins": 0.301483690738678,
      "rewards/rejected": -0.3066052496433258,
      "step": 530
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.52353616532721e-07,
      "logits/chosen": -2.4376657009124756,
      "logits/rejected": -2.4150702953338623,
      "logps/chosen": -266.14349365234375,
      "logps/rejected": -235.13882446289062,
      "loss": 0.6018,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.014219239354133606,
      "rewards/margins": 0.26282960176467896,
      "rewards/rejected": -0.27704882621765137,
      "step": 540
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.5044010715652504e-07,
      "logits/chosen": -2.4388842582702637,
      "logits/rejected": -2.390779495239258,
      "logps/chosen": -249.3515167236328,
      "logps/rejected": -228.97885131835938,
      "loss": 0.6,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.028151463717222214,
      "rewards/margins": 0.2760944068431854,
      "rewards/rejected": -0.30424588918685913,
      "step": 550
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.485265977803291e-07,
      "logits/chosen": -2.460728406906128,
      "logits/rejected": -2.3551411628723145,
      "logps/chosen": -293.16400146484375,
      "logps/rejected": -232.632080078125,
      "loss": 0.5871,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.06439458578824997,
      "rewards/margins": 0.4292263984680176,
      "rewards/rejected": -0.3648317754268646,
      "step": 560
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.4661308840413316e-07,
      "logits/chosen": -2.509218215942383,
      "logits/rejected": -2.375960111618042,
      "logps/chosen": -285.7981262207031,
      "logps/rejected": -236.2060546875,
      "loss": 0.6024,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.004936461336910725,
      "rewards/margins": 0.36126184463500977,
      "rewards/rejected": -0.3563253879547119,
      "step": 570
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.446995790279372e-07,
      "logits/chosen": -2.455012321472168,
      "logits/rejected": -2.436042547225952,
      "logps/chosen": -283.45404052734375,
      "logps/rejected": -230.90603637695312,
      "loss": 0.586,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.004780585877597332,
      "rewards/margins": 0.32074013352394104,
      "rewards/rejected": -0.31595954298973083,
      "step": 580
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.4278606965174123e-07,
      "logits/chosen": -2.4725747108459473,
      "logits/rejected": -2.470695734024048,
      "logps/chosen": -262.63873291015625,
      "logps/rejected": -241.70535278320312,
      "loss": 0.5869,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.011715828441083431,
      "rewards/margins": 0.34255507588386536,
      "rewards/rejected": -0.35427090525627136,
      "step": 590
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.408725602755453e-07,
      "logits/chosen": -2.460839033126831,
      "logits/rejected": -2.407060146331787,
      "logps/chosen": -233.80014038085938,
      "logps/rejected": -207.22445678710938,
      "loss": 0.5908,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03514878451824188,
      "rewards/margins": 0.27000272274017334,
      "rewards/rejected": -0.3051515221595764,
      "step": 600
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.3895905089934936e-07,
      "logits/chosen": -2.4760079383850098,
      "logits/rejected": -2.3460214138031006,
      "logps/chosen": -276.608154296875,
      "logps/rejected": -231.1323699951172,
      "loss": 0.59,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.0008779823547229171,
      "rewards/margins": 0.3627086579799652,
      "rewards/rejected": -0.3618307411670685,
      "step": 610
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.370455415231534e-07,
      "logits/chosen": -2.4187588691711426,
      "logits/rejected": -2.426133632659912,
      "logps/chosen": -276.6905212402344,
      "logps/rejected": -260.84893798828125,
      "loss": 0.5706,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05571802332997322,
      "rewards/margins": 0.310151606798172,
      "rewards/rejected": -0.3658696413040161,
      "step": 620
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.351320321469575e-07,
      "logits/chosen": -2.4383225440979004,
      "logits/rejected": -2.401784658432007,
      "logps/chosen": -298.56390380859375,
      "logps/rejected": -235.006591796875,
      "loss": 0.5838,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.035277482122182846,
      "rewards/margins": 0.44551342725753784,
      "rewards/rejected": -0.4102359712123871,
      "step": 630
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.3321852277076154e-07,
      "logits/chosen": -2.4030394554138184,
      "logits/rejected": -2.390498638153076,
      "logps/chosen": -267.2048645019531,
      "logps/rejected": -240.0560760498047,
      "loss": 0.5693,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.008936256170272827,
      "rewards/margins": 0.3941975235939026,
      "rewards/rejected": -0.3852612376213074,
      "step": 640
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.313050133945656e-07,
      "logits/chosen": -2.508209705352783,
      "logits/rejected": -2.45387601852417,
      "logps/chosen": -291.5556640625,
      "logps/rejected": -241.94131469726562,
      "loss": 0.5804,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.0002497173845767975,
      "rewards/margins": 0.4533967971801758,
      "rewards/rejected": -0.4536465108394623,
      "step": 650
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.2939150401836967e-07,
      "logits/chosen": -2.4588632583618164,
      "logits/rejected": -2.461416482925415,
      "logps/chosen": -245.1895751953125,
      "logps/rejected": -246.0583038330078,
      "loss": 0.5693,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06281181424856186,
      "rewards/margins": 0.3227921426296234,
      "rewards/rejected": -0.38560396432876587,
      "step": 660
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.2747799464217373e-07,
      "logits/chosen": -2.397804021835327,
      "logits/rejected": -2.3986358642578125,
      "logps/chosen": -262.5384521484375,
      "logps/rejected": -231.0225830078125,
      "loss": 0.5746,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.10527386516332626,
      "rewards/margins": 0.3180972933769226,
      "rewards/rejected": -0.42337116599082947,
      "step": 670
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.255644852659778e-07,
      "logits/chosen": -2.49613618850708,
      "logits/rejected": -2.4438865184783936,
      "logps/chosen": -287.05419921875,
      "logps/rejected": -249.3083953857422,
      "loss": 0.6131,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08329292386770248,
      "rewards/margins": 0.29655909538269043,
      "rewards/rejected": -0.3798519968986511,
      "step": 680
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.236509758897818e-07,
      "logits/chosen": -2.3793463706970215,
      "logits/rejected": -2.387934684753418,
      "logps/chosen": -260.1773986816406,
      "logps/rejected": -218.5209197998047,
      "loss": 0.602,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06579776108264923,
      "rewards/margins": 0.35870271921157837,
      "rewards/rejected": -0.4245004653930664,
      "step": 690
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.2173746651358586e-07,
      "logits/chosen": -2.412266254425049,
      "logits/rejected": -2.389812469482422,
      "logps/chosen": -260.27349853515625,
      "logps/rejected": -246.9514923095703,
      "loss": 0.5861,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10205063968896866,
      "rewards/margins": 0.2957732677459717,
      "rewards/rejected": -0.39782392978668213,
      "step": 700
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.198239571373899e-07,
      "logits/chosen": -2.42134690284729,
      "logits/rejected": -2.353253126144409,
      "logps/chosen": -243.3867645263672,
      "logps/rejected": -195.7749481201172,
      "loss": 0.5847,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05715983361005783,
      "rewards/margins": 0.43060773611068726,
      "rewards/rejected": -0.48776760697364807,
      "step": 710
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.17910447761194e-07,
      "logits/chosen": -2.4673993587493896,
      "logits/rejected": -2.3654751777648926,
      "logps/chosen": -267.1549377441406,
      "logps/rejected": -255.4532012939453,
      "loss": 0.5956,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08059550076723099,
      "rewards/margins": 0.34076714515686035,
      "rewards/rejected": -0.42136263847351074,
      "step": 720
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.1599693838499805e-07,
      "logits/chosen": -2.45065975189209,
      "logits/rejected": -2.4266138076782227,
      "logps/chosen": -295.47137451171875,
      "logps/rejected": -215.19296264648438,
      "loss": 0.5811,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.061896733939647675,
      "rewards/margins": 0.3598650097846985,
      "rewards/rejected": -0.42176175117492676,
      "step": 730
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.140834290088021e-07,
      "logits/chosen": -2.4394631385803223,
      "logits/rejected": -2.417912006378174,
      "logps/chosen": -277.1748046875,
      "logps/rejected": -221.8069610595703,
      "loss": 0.5631,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08770271390676498,
      "rewards/margins": 0.44523605704307556,
      "rewards/rejected": -0.5329388380050659,
      "step": 740
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.121699196326062e-07,
      "logits/chosen": -2.435689926147461,
      "logits/rejected": -2.310481548309326,
      "logps/chosen": -218.2103729248047,
      "logps/rejected": -185.0491943359375,
      "loss": 0.5802,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.05216900631785393,
      "rewards/margins": 0.38854002952575684,
      "rewards/rejected": -0.44070905447006226,
      "step": 750
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.1025641025641024e-07,
      "logits/chosen": -2.399353504180908,
      "logits/rejected": -2.4066669940948486,
      "logps/chosen": -259.16229248046875,
      "logps/rejected": -243.228515625,
      "loss": 0.5769,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06289193779230118,
      "rewards/margins": 0.4069492220878601,
      "rewards/rejected": -0.4698411524295807,
      "step": 760
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.083429008802143e-07,
      "logits/chosen": -2.525341510772705,
      "logits/rejected": -2.471456527709961,
      "logps/chosen": -274.21099853515625,
      "logps/rejected": -213.09451293945312,
      "loss": 0.5545,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.041963692754507065,
      "rewards/margins": 0.450117826461792,
      "rewards/rejected": -0.4920814633369446,
      "step": 770
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.0642939150401836e-07,
      "logits/chosen": -2.487657070159912,
      "logits/rejected": -2.3538174629211426,
      "logps/chosen": -290.19036865234375,
      "logps/rejected": -217.69906616210938,
      "loss": 0.5707,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06529206037521362,
      "rewards/margins": 0.44091683626174927,
      "rewards/rejected": -0.5062088966369629,
      "step": 780
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.0451588212782237e-07,
      "logits/chosen": -2.429466724395752,
      "logits/rejected": -2.395692825317383,
      "logps/chosen": -255.69052124023438,
      "logps/rejected": -227.69168090820312,
      "loss": 0.5409,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0646648183465004,
      "rewards/margins": 0.5062236189842224,
      "rewards/rejected": -0.5708884596824646,
      "step": 790
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0260237275162643e-07,
      "logits/chosen": -2.368137836456299,
      "logits/rejected": -2.3667895793914795,
      "logps/chosen": -232.4878692626953,
      "logps/rejected": -227.72323608398438,
      "loss": 0.5741,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.057848911732435226,
      "rewards/margins": 0.45716094970703125,
      "rewards/rejected": -0.5150099396705627,
      "step": 800
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.006888633754305e-07,
      "logits/chosen": -2.5232253074645996,
      "logits/rejected": -2.4718704223632812,
      "logps/chosen": -277.5395812988281,
      "logps/rejected": -229.75863647460938,
      "loss": 0.545,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02210831269621849,
      "rewards/margins": 0.5734663605690002,
      "rewards/rejected": -0.5955746173858643,
      "step": 810
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.9877535399923456e-07,
      "logits/chosen": -2.4078879356384277,
      "logits/rejected": -2.3697750568389893,
      "logps/chosen": -267.1474914550781,
      "logps/rejected": -224.79318237304688,
      "loss": 0.5611,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05940120667219162,
      "rewards/margins": 0.5237945318222046,
      "rewards/rejected": -0.583195686340332,
      "step": 820
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.968618446230386e-07,
      "logits/chosen": -2.441854953765869,
      "logits/rejected": -2.413933277130127,
      "logps/chosen": -319.1496276855469,
      "logps/rejected": -247.3388671875,
      "loss": 0.5411,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.00216462230309844,
      "rewards/margins": 0.6005392074584961,
      "rewards/rejected": -0.6027037501335144,
      "step": 830
    },
    {
      "epoch": 0.87,
      "learning_rate": 3.949483352468427e-07,
      "logits/chosen": -2.441795825958252,
      "logits/rejected": -2.408433437347412,
      "logps/chosen": -275.55364990234375,
      "logps/rejected": -240.1023712158203,
      "loss": 0.5996,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.18812677264213562,
      "rewards/margins": 0.3052147328853607,
      "rewards/rejected": -0.4933415353298187,
      "step": 840
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.9303482587064674e-07,
      "logits/chosen": -2.4030299186706543,
      "logits/rejected": -2.4201598167419434,
      "logps/chosen": -282.9662170410156,
      "logps/rejected": -238.91635131835938,
      "loss": 0.5653,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07580298185348511,
      "rewards/margins": 0.48956719040870667,
      "rewards/rejected": -0.5653700828552246,
      "step": 850
    },
    {
      "epoch": 0.89,
      "learning_rate": 3.911213164944508e-07,
      "logits/chosen": -2.4482102394104004,
      "logits/rejected": -2.3696365356445312,
      "logps/chosen": -291.52374267578125,
      "logps/rejected": -245.3708953857422,
      "loss": 0.5645,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0693991631269455,
      "rewards/margins": 0.5479902029037476,
      "rewards/rejected": -0.6173893809318542,
      "step": 860
    },
    {
      "epoch": 0.9,
      "learning_rate": 3.8920780711825487e-07,
      "logits/chosen": -2.333220958709717,
      "logits/rejected": -2.36674165725708,
      "logps/chosen": -270.8648376464844,
      "logps/rejected": -226.4284210205078,
      "loss": 0.5623,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07031848281621933,
      "rewards/margins": 0.5230459570884705,
      "rewards/rejected": -0.5933644771575928,
      "step": 870
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.8729429774205893e-07,
      "logits/chosen": -2.414712905883789,
      "logits/rejected": -2.3959202766418457,
      "logps/chosen": -265.5609436035156,
      "logps/rejected": -221.8790283203125,
      "loss": 0.5583,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09441053122282028,
      "rewards/margins": 0.4818418025970459,
      "rewards/rejected": -0.5762523412704468,
      "step": 880
    },
    {
      "epoch": 0.92,
      "learning_rate": 3.8538078836586294e-07,
      "logits/chosen": -2.4260947704315186,
      "logits/rejected": -2.3521246910095215,
      "logps/chosen": -255.67459106445312,
      "logps/rejected": -249.5232696533203,
      "loss": 0.5798,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.11774387210607529,
      "rewards/margins": 0.41275423765182495,
      "rewards/rejected": -0.5304980874061584,
      "step": 890
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.83467278989667e-07,
      "logits/chosen": -2.4081029891967773,
      "logits/rejected": -2.356966495513916,
      "logps/chosen": -282.0208740234375,
      "logps/rejected": -222.43765258789062,
      "loss": 0.5496,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0056116655468940735,
      "rewards/margins": 0.560874879360199,
      "rewards/rejected": -0.5664864778518677,
      "step": 900
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.8155376961347106e-07,
      "logits/chosen": -2.4205422401428223,
      "logits/rejected": -2.4086201190948486,
      "logps/chosen": -250.5408477783203,
      "logps/rejected": -203.41635131835938,
      "loss": 0.5658,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.08207114040851593,
      "rewards/margins": 0.5275572538375854,
      "rewards/rejected": -0.6096283197402954,
      "step": 910
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.796402602372751e-07,
      "logits/chosen": -2.4266042709350586,
      "logits/rejected": -2.3435492515563965,
      "logps/chosen": -272.265380859375,
      "logps/rejected": -252.53268432617188,
      "loss": 0.5561,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08728349208831787,
      "rewards/margins": 0.43014270067214966,
      "rewards/rejected": -0.5174261927604675,
      "step": 920
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.777267508610792e-07,
      "logits/chosen": -2.4939112663269043,
      "logits/rejected": -2.428483247756958,
      "logps/chosen": -270.8354797363281,
      "logps/rejected": -260.45611572265625,
      "loss": 0.5662,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.1041080579161644,
      "rewards/margins": 0.5785307884216309,
      "rewards/rejected": -0.6826388239860535,
      "step": 930
    },
    {
      "epoch": 0.97,
      "learning_rate": 3.7581324148488325e-07,
      "logits/chosen": -2.4214115142822266,
      "logits/rejected": -2.374481201171875,
      "logps/chosen": -299.26385498046875,
      "logps/rejected": -224.3602294921875,
      "loss": 0.5676,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.09809824079275131,
      "rewards/margins": 0.4741215705871582,
      "rewards/rejected": -0.5722197890281677,
      "step": 940
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.738997321086873e-07,
      "logits/chosen": -2.405039072036743,
      "logits/rejected": -2.36025333404541,
      "logps/chosen": -257.98358154296875,
      "logps/rejected": -224.39584350585938,
      "loss": 0.5705,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.055282533168792725,
      "rewards/margins": 0.41767168045043945,
      "rewards/rejected": -0.4729541838169098,
      "step": 950
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.7198622273249137e-07,
      "logits/chosen": -2.4322896003723145,
      "logits/rejected": -2.4136605262756348,
      "logps/chosen": -262.254150390625,
      "logps/rejected": -195.317626953125,
      "loss": 0.5501,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.09157253801822662,
      "rewards/margins": 0.4116409420967102,
      "rewards/rejected": -0.5032135248184204,
      "step": 960
    },
    {
      "epoch": 1.0,
      "eval_logits/chosen": -2.1115331649780273,
      "eval_logits/rejected": -1.9901020526885986,
      "eval_logps/chosen": -265.7538146972656,
      "eval_logps/rejected": -225.6846923828125,
      "eval_loss": 0.5559706091880798,
      "eval_rewards/accuracies": 0.7200000286102295,
      "eval_rewards/chosen": -0.10930197685956955,
      "eval_rewards/margins": 0.5307366251945496,
      "eval_rewards/rejected": -0.6400385499000549,
      "eval_runtime": 454.9548,
      "eval_samples_per_second": 4.396,
      "eval_steps_per_second": 0.275,
      "step": 968
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.7007271335629544e-07,
      "logits/chosen": -2.393294334411621,
      "logits/rejected": -2.2951369285583496,
      "logps/chosen": -265.35040283203125,
      "logps/rejected": -216.5760955810547,
      "loss": 0.5536,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0699644461274147,
      "rewards/margins": 0.6380785703659058,
      "rewards/rejected": -0.7080430388450623,
      "step": 970
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.681592039800995e-07,
      "logits/chosen": -2.384970188140869,
      "logits/rejected": -2.355285167694092,
      "logps/chosen": -242.12600708007812,
      "logps/rejected": -217.994873046875,
      "loss": 0.5518,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.10410074144601822,
      "rewards/margins": 0.4534371495246887,
      "rewards/rejected": -0.557537853717804,
      "step": 980
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.662456946039035e-07,
      "logits/chosen": -2.3399007320404053,
      "logits/rejected": -2.3183391094207764,
      "logps/chosen": -231.69064331054688,
      "logps/rejected": -201.68441772460938,
      "loss": 0.5997,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2032332420349121,
      "rewards/margins": 0.3764384686946869,
      "rewards/rejected": -0.5796716809272766,
      "step": 990
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.6433218522770757e-07,
      "logits/chosen": -2.3833537101745605,
      "logits/rejected": -2.319551944732666,
      "logps/chosen": -295.35052490234375,
      "logps/rejected": -220.7740936279297,
      "loss": 0.5815,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07751850038766861,
      "rewards/margins": 0.4878837466239929,
      "rewards/rejected": -0.5654021501541138,
      "step": 1000
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.6241867585151163e-07,
      "logits/chosen": -2.449373245239258,
      "logits/rejected": -2.387481927871704,
      "logps/chosen": -248.5247802734375,
      "logps/rejected": -225.5816192626953,
      "loss": 0.5582,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.13464747369289398,
      "rewards/margins": 0.4785802960395813,
      "rewards/rejected": -0.6132277250289917,
      "step": 1010
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.605051664753157e-07,
      "logits/chosen": -2.4245686531066895,
      "logits/rejected": -2.4296786785125732,
      "logps/chosen": -251.46206665039062,
      "logps/rejected": -202.94284057617188,
      "loss": 0.5488,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.10227508842945099,
      "rewards/margins": 0.5419632792472839,
      "rewards/rejected": -0.6442384123802185,
      "step": 1020
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.5859165709911975e-07,
      "logits/chosen": -2.3132331371307373,
      "logits/rejected": -2.3190786838531494,
      "logps/chosen": -292.53125,
      "logps/rejected": -256.5845947265625,
      "loss": 0.5407,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.04681607708334923,
      "rewards/margins": 0.629582941532135,
      "rewards/rejected": -0.6763990521430969,
      "step": 1030
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.566781477229238e-07,
      "logits/chosen": -2.36743426322937,
      "logits/rejected": -2.2601940631866455,
      "logps/chosen": -259.8556213378906,
      "logps/rejected": -235.2071075439453,
      "loss": 0.5544,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0872168317437172,
      "rewards/margins": 0.5171287059783936,
      "rewards/rejected": -0.6043455600738525,
      "step": 1040
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.547646383467279e-07,
      "logits/chosen": -2.326066732406616,
      "logits/rejected": -2.3307995796203613,
      "logps/chosen": -256.3676452636719,
      "logps/rejected": -235.98672485351562,
      "loss": 0.528,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.039658013731241226,
      "rewards/margins": 0.6266075372695923,
      "rewards/rejected": -0.6662655472755432,
      "step": 1050
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.5285112897053194e-07,
      "logits/chosen": -2.4012277126312256,
      "logits/rejected": -2.287254810333252,
      "logps/chosen": -275.3678894042969,
      "logps/rejected": -235.6710662841797,
      "loss": 0.5394,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1649123877286911,
      "rewards/margins": 0.7132270932197571,
      "rewards/rejected": -0.8781394958496094,
      "step": 1060
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.50937619594336e-07,
      "logits/chosen": -2.4277257919311523,
      "logits/rejected": -2.3757331371307373,
      "logps/chosen": -291.19110107421875,
      "logps/rejected": -261.84320068359375,
      "loss": 0.5486,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08061722666025162,
      "rewards/margins": 0.646703839302063,
      "rewards/rejected": -0.7273210883140564,
      "step": 1070
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.4902411021814007e-07,
      "logits/chosen": -2.4494857788085938,
      "logits/rejected": -2.3594818115234375,
      "logps/chosen": -294.79901123046875,
      "logps/rejected": -229.1653594970703,
      "loss": 0.5273,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0071506015956401825,
      "rewards/margins": 0.7936251759529114,
      "rewards/rejected": -0.8007757067680359,
      "step": 1080
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.4711060084194413e-07,
      "logits/chosen": -2.3939168453216553,
      "logits/rejected": -2.403517961502075,
      "logps/chosen": -272.8337707519531,
      "logps/rejected": -228.14816284179688,
      "loss": 0.569,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.05292476341128349,
      "rewards/margins": 0.5845643281936646,
      "rewards/rejected": -0.6374891996383667,
      "step": 1090
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.4519709146574814e-07,
      "logits/chosen": -2.4750289916992188,
      "logits/rejected": -2.3406853675842285,
      "logps/chosen": -267.2197265625,
      "logps/rejected": -247.76498413085938,
      "loss": 0.5588,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07975287735462189,
      "rewards/margins": 0.5557757019996643,
      "rewards/rejected": -0.6355286240577698,
      "step": 1100
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.432835820895522e-07,
      "logits/chosen": -2.4085628986358643,
      "logits/rejected": -2.3299522399902344,
      "logps/chosen": -250.01791381835938,
      "logps/rejected": -230.2222900390625,
      "loss": 0.5427,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.09803862869739532,
      "rewards/margins": 0.5516743659973145,
      "rewards/rejected": -0.6497129797935486,
      "step": 1110
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.4137007271335626e-07,
      "logits/chosen": -2.349724054336548,
      "logits/rejected": -2.396789073944092,
      "logps/chosen": -218.85134887695312,
      "logps/rejected": -212.79843139648438,
      "loss": 0.5405,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.17577314376831055,
      "rewards/margins": 0.4191734790802002,
      "rewards/rejected": -0.5949466228485107,
      "step": 1120
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.394565633371603e-07,
      "logits/chosen": -2.3591291904449463,
      "logits/rejected": -2.2995028495788574,
      "logps/chosen": -293.353271484375,
      "logps/rejected": -245.3329315185547,
      "loss": 0.5378,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1506257951259613,
      "rewards/margins": 0.5927566289901733,
      "rewards/rejected": -0.7433823347091675,
      "step": 1130
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.375430539609644e-07,
      "logits/chosen": -2.281580924987793,
      "logits/rejected": -2.3465418815612793,
      "logps/chosen": -239.721923828125,
      "logps/rejected": -211.04336547851562,
      "loss": 0.5148,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.13178768754005432,
      "rewards/margins": 0.6117417812347412,
      "rewards/rejected": -0.7435294985771179,
      "step": 1140
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.3562954458476845e-07,
      "logits/chosen": -2.4424147605895996,
      "logits/rejected": -2.428922653198242,
      "logps/chosen": -306.62567138671875,
      "logps/rejected": -233.6839141845703,
      "loss": 0.5608,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.14439252018928528,
      "rewards/margins": 0.5881448984146118,
      "rewards/rejected": -0.7325373888015747,
      "step": 1150
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.337160352085725e-07,
      "logits/chosen": -2.4268736839294434,
      "logits/rejected": -2.385798692703247,
      "logps/chosen": -298.24273681640625,
      "logps/rejected": -241.7369842529297,
      "loss": 0.535,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06826777756214142,
      "rewards/margins": 0.7597671747207642,
      "rewards/rejected": -0.8280348777770996,
      "step": 1160
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.3180252583237657e-07,
      "logits/chosen": -2.3368301391601562,
      "logits/rejected": -2.2904515266418457,
      "logps/chosen": -277.94744873046875,
      "logps/rejected": -231.4571533203125,
      "loss": 0.5509,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.057608962059020996,
      "rewards/margins": 0.6012957692146301,
      "rewards/rejected": -0.6589046716690063,
      "step": 1170
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.2988901645618063e-07,
      "logits/chosen": -2.3686211109161377,
      "logits/rejected": -2.278508424758911,
      "logps/chosen": -261.26556396484375,
      "logps/rejected": -225.91049194335938,
      "loss": 0.5406,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1395711600780487,
      "rewards/margins": 0.5758097171783447,
      "rewards/rejected": -0.715380847454071,
      "step": 1180
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.279755070799847e-07,
      "logits/chosen": -2.402057647705078,
      "logits/rejected": -2.310411214828491,
      "logps/chosen": -259.6847229003906,
      "logps/rejected": -257.0271301269531,
      "loss": 0.5279,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1509382575750351,
      "rewards/margins": 0.5303701162338257,
      "rewards/rejected": -0.681308388710022,
      "step": 1190
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.260619977037887e-07,
      "logits/chosen": -2.397024154663086,
      "logits/rejected": -2.3406670093536377,
      "logps/chosen": -251.27926635742188,
      "logps/rejected": -214.0767822265625,
      "loss": 0.5191,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07334662973880768,
      "rewards/margins": 0.7737401723861694,
      "rewards/rejected": -0.8470869064331055,
      "step": 1200
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.2414848832759277e-07,
      "logits/chosen": -2.42278790473938,
      "logits/rejected": -2.2710700035095215,
      "logps/chosen": -261.6346435546875,
      "logps/rejected": -227.0576171875,
      "loss": 0.5345,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.13957397639751434,
      "rewards/margins": 0.585122287273407,
      "rewards/rejected": -0.7246962785720825,
      "step": 1210
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.2223497895139683e-07,
      "logits/chosen": -2.3876070976257324,
      "logits/rejected": -2.3234522342681885,
      "logps/chosen": -263.7557678222656,
      "logps/rejected": -219.36367797851562,
      "loss": 0.5538,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12251804769039154,
      "rewards/margins": 0.5337706804275513,
      "rewards/rejected": -0.656288743019104,
      "step": 1220
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.203214695752009e-07,
      "logits/chosen": -2.4647679328918457,
      "logits/rejected": -2.385979175567627,
      "logps/chosen": -262.8483581542969,
      "logps/rejected": -230.214111328125,
      "loss": 0.5692,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15525886416435242,
      "rewards/margins": 0.5802783370018005,
      "rewards/rejected": -0.7355371713638306,
      "step": 1230
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.1840796019900495e-07,
      "logits/chosen": -2.4262988567352295,
      "logits/rejected": -2.3878419399261475,
      "logps/chosen": -262.4114074707031,
      "logps/rejected": -255.02749633789062,
      "loss": 0.5478,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.11712455749511719,
      "rewards/margins": 0.6756891012191772,
      "rewards/rejected": -0.7928137183189392,
      "step": 1240
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.16494450822809e-07,
      "logits/chosen": -2.35296893119812,
      "logits/rejected": -2.3551628589630127,
      "logps/chosen": -236.53445434570312,
      "logps/rejected": -233.4728546142578,
      "loss": 0.5355,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.18067090213298798,
      "rewards/margins": 0.4853228032588959,
      "rewards/rejected": -0.6659936308860779,
      "step": 1250
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.145809414466131e-07,
      "logits/chosen": -2.48589825630188,
      "logits/rejected": -2.478086233139038,
      "logps/chosen": -272.775634765625,
      "logps/rejected": -266.91058349609375,
      "loss": 0.5399,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.14183524250984192,
      "rewards/margins": 0.4844776690006256,
      "rewards/rejected": -0.6263129711151123,
      "step": 1260
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.1266743207041714e-07,
      "logits/chosen": -2.406167507171631,
      "logits/rejected": -2.4039671421051025,
      "logps/chosen": -247.5933074951172,
      "logps/rejected": -226.58450317382812,
      "loss": 0.5439,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.1417725682258606,
      "rewards/margins": 0.6144998073577881,
      "rewards/rejected": -0.7562723755836487,
      "step": 1270
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.107539226942212e-07,
      "logits/chosen": -2.392794132232666,
      "logits/rejected": -2.370959758758545,
      "logps/chosen": -259.82183837890625,
      "logps/rejected": -240.86178588867188,
      "loss": 0.5788,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.14074555039405823,
      "rewards/margins": 0.5021336674690247,
      "rewards/rejected": -0.6428791880607605,
      "step": 1280
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.0884041331802526e-07,
      "logits/chosen": -2.3568851947784424,
      "logits/rejected": -2.2625174522399902,
      "logps/chosen": -243.117431640625,
      "logps/rejected": -216.640380859375,
      "loss": 0.5542,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.17768898606300354,
      "rewards/margins": 0.5181851983070374,
      "rewards/rejected": -0.6958742141723633,
      "step": 1290
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.0692690394182927e-07,
      "logits/chosen": -2.436131477355957,
      "logits/rejected": -2.396608829498291,
      "logps/chosen": -277.2726135253906,
      "logps/rejected": -233.1066131591797,
      "loss": 0.5558,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06510978192090988,
      "rewards/margins": 0.6768725514411926,
      "rewards/rejected": -0.7419822812080383,
      "step": 1300
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.0501339456563334e-07,
      "logits/chosen": -2.4156711101531982,
      "logits/rejected": -2.271003007888794,
      "logps/chosen": -269.3404541015625,
      "logps/rejected": -243.42245483398438,
      "loss": 0.534,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.16720613837242126,
      "rewards/margins": 0.5710007548332214,
      "rewards/rejected": -0.7382069230079651,
      "step": 1310
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.030998851894374e-07,
      "logits/chosen": -2.352940082550049,
      "logits/rejected": -2.363243579864502,
      "logps/chosen": -229.7804718017578,
      "logps/rejected": -221.0922393798828,
      "loss": 0.5117,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.08244588226079941,
      "rewards/margins": 0.668733537197113,
      "rewards/rejected": -0.7511794567108154,
      "step": 1320
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.0118637581324146e-07,
      "logits/chosen": -2.4218077659606934,
      "logits/rejected": -2.36722469329834,
      "logps/chosen": -257.25872802734375,
      "logps/rejected": -223.78256225585938,
      "loss": 0.539,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.08728824555873871,
      "rewards/margins": 0.7230480909347534,
      "rewards/rejected": -0.8103362917900085,
      "step": 1330
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.992728664370455e-07,
      "logits/chosen": -2.4494032859802246,
      "logits/rejected": -2.388096332550049,
      "logps/chosen": -239.74057006835938,
      "logps/rejected": -225.5271453857422,
      "loss": 0.5603,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.21483702957630157,
      "rewards/margins": 0.5343385338783264,
      "rewards/rejected": -0.7491755485534668,
      "step": 1340
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.973593570608496e-07,
      "logits/chosen": -2.3430349826812744,
      "logits/rejected": -2.262885808944702,
      "logps/chosen": -261.22149658203125,
      "logps/rejected": -220.1973419189453,
      "loss": 0.5515,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11469510942697525,
      "rewards/margins": 0.6628144979476929,
      "rewards/rejected": -0.7775096297264099,
      "step": 1350
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.9544584768465365e-07,
      "logits/chosen": -2.415313243865967,
      "logits/rejected": -2.369680881500244,
      "logps/chosen": -310.0767517089844,
      "logps/rejected": -254.2592315673828,
      "loss": 0.5553,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.1948031485080719,
      "rewards/margins": 0.6389765739440918,
      "rewards/rejected": -0.8337796926498413,
      "step": 1360
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.935323383084577e-07,
      "logits/chosen": -2.3823018074035645,
      "logits/rejected": -2.3315823078155518,
      "logps/chosen": -290.1065368652344,
      "logps/rejected": -242.30081176757812,
      "loss": 0.5604,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.09571398794651031,
      "rewards/margins": 0.5208708643913269,
      "rewards/rejected": -0.6165848970413208,
      "step": 1370
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.9161882893226177e-07,
      "logits/chosen": -2.3578696250915527,
      "logits/rejected": -2.2521920204162598,
      "logps/chosen": -240.7886962890625,
      "logps/rejected": -204.34652709960938,
      "loss": 0.544,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.20477895438671112,
      "rewards/margins": 0.4378456473350525,
      "rewards/rejected": -0.6426246166229248,
      "step": 1380
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.8970531955606583e-07,
      "logits/chosen": -2.453507900238037,
      "logits/rejected": -2.3540263175964355,
      "logps/chosen": -270.1062927246094,
      "logps/rejected": -231.31466674804688,
      "loss": 0.5674,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.16483107209205627,
      "rewards/margins": 0.64717036485672,
      "rewards/rejected": -0.8120015263557434,
      "step": 1390
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.8779181017986984e-07,
      "logits/chosen": -2.424034595489502,
      "logits/rejected": -2.4023849964141846,
      "logps/chosen": -269.05780029296875,
      "logps/rejected": -239.2170867919922,
      "loss": 0.5546,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.22925500571727753,
      "rewards/margins": 0.5416269898414612,
      "rewards/rejected": -0.7708818912506104,
      "step": 1400
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.858783008036739e-07,
      "logits/chosen": -2.3142106533050537,
      "logits/rejected": -2.261690616607666,
      "logps/chosen": -278.53643798828125,
      "logps/rejected": -220.97555541992188,
      "loss": 0.5294,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.13496367633342743,
      "rewards/margins": 0.6569581031799316,
      "rewards/rejected": -0.7919217944145203,
      "step": 1410
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.8396479142747797e-07,
      "logits/chosen": -2.3985562324523926,
      "logits/rejected": -2.368497848510742,
      "logps/chosen": -230.22122192382812,
      "logps/rejected": -201.47998046875,
      "loss": 0.5405,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1572544276714325,
      "rewards/margins": 0.6025274991989136,
      "rewards/rejected": -0.7597819566726685,
      "step": 1420
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.8205128205128203e-07,
      "logits/chosen": -2.3532938957214355,
      "logits/rejected": -2.30509352684021,
      "logps/chosen": -260.51312255859375,
      "logps/rejected": -229.70376586914062,
      "loss": 0.5412,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2625270187854767,
      "rewards/margins": 0.5138301253318787,
      "rewards/rejected": -0.776357114315033,
      "step": 1430
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.801377726750861e-07,
      "logits/chosen": -2.3864636421203613,
      "logits/rejected": -2.316066265106201,
      "logps/chosen": -256.13763427734375,
      "logps/rejected": -217.63235473632812,
      "loss": 0.5431,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1825113743543625,
      "rewards/margins": 0.49288827180862427,
      "rewards/rejected": -0.675399661064148,
      "step": 1440
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.7822426329889015e-07,
      "logits/chosen": -2.428596019744873,
      "logits/rejected": -2.3228797912597656,
      "logps/chosen": -294.236328125,
      "logps/rejected": -235.548095703125,
      "loss": 0.5636,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.1488821804523468,
      "rewards/margins": 0.5950523018836975,
      "rewards/rejected": -0.7439345121383667,
      "step": 1450
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.763107539226942e-07,
      "logits/chosen": -2.380725145339966,
      "logits/rejected": -2.3312487602233887,
      "logps/chosen": -280.84490966796875,
      "logps/rejected": -242.58340454101562,
      "loss": 0.5586,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.13354410231113434,
      "rewards/margins": 0.6448094844818115,
      "rewards/rejected": -0.7783535718917847,
      "step": 1460
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.743972445464983e-07,
      "logits/chosen": -2.4229865074157715,
      "logits/rejected": -2.410762071609497,
      "logps/chosen": -254.83706665039062,
      "logps/rejected": -232.5323486328125,
      "loss": 0.5598,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.1900000274181366,
      "rewards/margins": 0.4888847768306732,
      "rewards/rejected": -0.678884744644165,
      "step": 1470
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.7248373517030234e-07,
      "logits/chosen": -2.475088119506836,
      "logits/rejected": -2.4195237159729004,
      "logps/chosen": -274.3759765625,
      "logps/rejected": -237.31689453125,
      "loss": 0.5401,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.16161084175109863,
      "rewards/margins": 0.6231001615524292,
      "rewards/rejected": -0.7847110629081726,
      "step": 1480
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.705702257941064e-07,
      "logits/chosen": -2.432378053665161,
      "logits/rejected": -2.4352517127990723,
      "logps/chosen": -311.6330261230469,
      "logps/rejected": -246.75796508789062,
      "loss": 0.5356,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.16773328185081482,
      "rewards/margins": 0.6822336912155151,
      "rewards/rejected": -0.8499670028686523,
      "step": 1490
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.686567164179104e-07,
      "logits/chosen": -2.348602771759033,
      "logits/rejected": -2.275367259979248,
      "logps/chosen": -259.00390625,
      "logps/rejected": -219.0964813232422,
      "loss": 0.5445,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.16603624820709229,
      "rewards/margins": 0.6049584150314331,
      "rewards/rejected": -0.7709946632385254,
      "step": 1500
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.6674320704171447e-07,
      "logits/chosen": -2.416759729385376,
      "logits/rejected": -2.3592278957366943,
      "logps/chosen": -259.57403564453125,
      "logps/rejected": -206.39962768554688,
      "loss": 0.5209,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.10807810723781586,
      "rewards/margins": 0.7587023973464966,
      "rewards/rejected": -0.8667804598808289,
      "step": 1510
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.6482969766551853e-07,
      "logits/chosen": -2.4495720863342285,
      "logits/rejected": -2.447631359100342,
      "logps/chosen": -274.3368225097656,
      "logps/rejected": -237.6719970703125,
      "loss": 0.5443,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.08986170589923859,
      "rewards/margins": 0.7171138525009155,
      "rewards/rejected": -0.8069756627082825,
      "step": 1520
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.629161882893226e-07,
      "logits/chosen": -2.4459540843963623,
      "logits/rejected": -2.371903419494629,
      "logps/chosen": -280.4138488769531,
      "logps/rejected": -214.16226196289062,
      "loss": 0.5159,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.15295711159706116,
      "rewards/margins": 0.7536977529525757,
      "rewards/rejected": -0.906654953956604,
      "step": 1530
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.6100267891312666e-07,
      "logits/chosen": -2.3349404335021973,
      "logits/rejected": -2.303260087966919,
      "logps/chosen": -269.13787841796875,
      "logps/rejected": -208.63107299804688,
      "loss": 0.5546,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12116660922765732,
      "rewards/margins": 0.6888124942779541,
      "rewards/rejected": -0.8099790811538696,
      "step": 1540
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.590891695369307e-07,
      "logits/chosen": -2.450225830078125,
      "logits/rejected": -2.361754894256592,
      "logps/chosen": -256.0200500488281,
      "logps/rejected": -210.36026000976562,
      "loss": 0.529,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1600186824798584,
      "rewards/margins": 0.5653270483016968,
      "rewards/rejected": -0.7253457903862,
      "step": 1550
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.571756601607348e-07,
      "logits/chosen": -2.4175140857696533,
      "logits/rejected": -2.3866875171661377,
      "logps/chosen": -294.81744384765625,
      "logps/rejected": -217.4511260986328,
      "loss": 0.5455,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.18921521306037903,
      "rewards/margins": 0.6538978815078735,
      "rewards/rejected": -0.843113124370575,
      "step": 1560
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.5526215078453884e-07,
      "logits/chosen": -2.4297268390655518,
      "logits/rejected": -2.3942902088165283,
      "logps/chosen": -301.64642333984375,
      "logps/rejected": -237.7666015625,
      "loss": 0.5349,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.10356118530035019,
      "rewards/margins": 0.7558127045631409,
      "rewards/rejected": -0.859373927116394,
      "step": 1570
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.533486414083429e-07,
      "logits/chosen": -2.4258458614349365,
      "logits/rejected": -2.3717398643493652,
      "logps/chosen": -306.97442626953125,
      "logps/rejected": -249.22109985351562,
      "loss": 0.5161,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07514607906341553,
      "rewards/margins": 0.7315629720687866,
      "rewards/rejected": -0.8067091107368469,
      "step": 1580
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.5143513203214697e-07,
      "logits/chosen": -2.3663864135742188,
      "logits/rejected": -2.326347827911377,
      "logps/chosen": -275.7830505371094,
      "logps/rejected": -225.2949676513672,
      "loss": 0.5606,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.20903043448925018,
      "rewards/margins": 0.5634015202522278,
      "rewards/rejected": -0.7724319696426392,
      "step": 1590
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.49521622655951e-07,
      "logits/chosen": -2.492893934249878,
      "logits/rejected": -2.3561127185821533,
      "logps/chosen": -293.12384033203125,
      "logps/rejected": -262.7926940917969,
      "loss": 0.553,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.04525697976350784,
      "rewards/margins": 0.8012422323226929,
      "rewards/rejected": -0.8464992642402649,
      "step": 1600
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.4760811327975504e-07,
      "logits/chosen": -2.4678902626037598,
      "logits/rejected": -2.3764843940734863,
      "logps/chosen": -273.65106201171875,
      "logps/rejected": -253.1365966796875,
      "loss": 0.5352,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.13813751935958862,
      "rewards/margins": 0.5831281542778015,
      "rewards/rejected": -0.7212656736373901,
      "step": 1610
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.456946039035591e-07,
      "logits/chosen": -2.466249942779541,
      "logits/rejected": -2.436295747756958,
      "logps/chosen": -290.6951599121094,
      "logps/rejected": -257.230712890625,
      "loss": 0.5668,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.10182980448007584,
      "rewards/margins": 0.7511130571365356,
      "rewards/rejected": -0.852942943572998,
      "step": 1620
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.4378109452736316e-07,
      "logits/chosen": -2.3190982341766357,
      "logits/rejected": -2.3256137371063232,
      "logps/chosen": -272.5343322753906,
      "logps/rejected": -198.2899932861328,
      "loss": 0.5291,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10047291219234467,
      "rewards/margins": 0.7110286951065063,
      "rewards/rejected": -0.8115016222000122,
      "step": 1630
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.418675851511672e-07,
      "logits/chosen": -2.3069424629211426,
      "logits/rejected": -2.312654495239258,
      "logps/chosen": -281.74322509765625,
      "logps/rejected": -221.7678680419922,
      "loss": 0.5624,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.17993247509002686,
      "rewards/margins": 0.6608012318611145,
      "rewards/rejected": -0.8407337069511414,
      "step": 1640
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.399540757749713e-07,
      "logits/chosen": -2.372399091720581,
      "logits/rejected": -2.2994461059570312,
      "logps/chosen": -250.9709014892578,
      "logps/rejected": -216.40042114257812,
      "loss": 0.5261,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07177358865737915,
      "rewards/margins": 0.9095686078071594,
      "rewards/rejected": -0.9813421964645386,
      "step": 1650
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.3804056639877535e-07,
      "logits/chosen": -2.333723545074463,
      "logits/rejected": -2.250431537628174,
      "logps/chosen": -269.04998779296875,
      "logps/rejected": -230.35256958007812,
      "loss": 0.5566,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.1327822506427765,
      "rewards/margins": 0.7238941192626953,
      "rewards/rejected": -0.8566763997077942,
      "step": 1660
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.361270570225794e-07,
      "logits/chosen": -2.3128609657287598,
      "logits/rejected": -2.2538418769836426,
      "logps/chosen": -263.09417724609375,
      "logps/rejected": -223.70974731445312,
      "loss": 0.5049,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.11414393037557602,
      "rewards/margins": 0.789380669593811,
      "rewards/rejected": -0.9035245776176453,
      "step": 1670
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.3421354764638345e-07,
      "logits/chosen": -2.4389374256134033,
      "logits/rejected": -2.3650295734405518,
      "logps/chosen": -296.4449462890625,
      "logps/rejected": -244.5340576171875,
      "loss": 0.554,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.16256913542747498,
      "rewards/margins": 0.6594110727310181,
      "rewards/rejected": -0.8219801783561707,
      "step": 1680
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.323000382701875e-07,
      "logits/chosen": -2.38859224319458,
      "logits/rejected": -2.275116443634033,
      "logps/chosen": -289.535400390625,
      "logps/rejected": -226.6898956298828,
      "loss": 0.5174,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11396412551403046,
      "rewards/margins": 0.6983034014701843,
      "rewards/rejected": -0.8122674822807312,
      "step": 1690
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.3038652889399157e-07,
      "logits/chosen": -2.3573622703552246,
      "logits/rejected": -2.368932008743286,
      "logps/chosen": -259.396240234375,
      "logps/rejected": -235.6220245361328,
      "loss": 0.5219,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0638931542634964,
      "rewards/margins": 0.7549349069595337,
      "rewards/rejected": -0.8188279867172241,
      "step": 1700
    },
    {
      "epoch": 1.77,
      "learning_rate": 2.2847301951779563e-07,
      "logits/chosen": -2.3194305896759033,
      "logits/rejected": -2.271104574203491,
      "logps/chosen": -230.3657989501953,
      "logps/rejected": -218.594970703125,
      "loss": 0.5364,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.23464179039001465,
      "rewards/margins": 0.5825188755989075,
      "rewards/rejected": -0.8171606063842773,
      "step": 1710
    },
    {
      "epoch": 1.78,
      "learning_rate": 2.265595101415997e-07,
      "logits/chosen": -2.3034956455230713,
      "logits/rejected": -2.26176118850708,
      "logps/chosen": -263.59234619140625,
      "logps/rejected": -207.0500946044922,
      "loss": 0.5405,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.1522129625082016,
      "rewards/margins": 0.8050564527511597,
      "rewards/rejected": -0.957269549369812,
      "step": 1720
    },
    {
      "epoch": 1.79,
      "learning_rate": 2.2464600076540373e-07,
      "logits/chosen": -2.3417623043060303,
      "logits/rejected": -2.3644707202911377,
      "logps/chosen": -272.0294494628906,
      "logps/rejected": -233.199951171875,
      "loss": 0.5376,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.14386221766471863,
      "rewards/margins": 0.7099758386611938,
      "rewards/rejected": -0.8538379669189453,
      "step": 1730
    },
    {
      "epoch": 1.8,
      "learning_rate": 2.227324913892078e-07,
      "logits/chosen": -2.4512619972229004,
      "logits/rejected": -2.333153247833252,
      "logps/chosen": -262.7388610839844,
      "logps/rejected": -233.94564819335938,
      "loss": 0.5274,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09474427998065948,
      "rewards/margins": 0.678414523601532,
      "rewards/rejected": -0.773158848285675,
      "step": 1740
    },
    {
      "epoch": 1.81,
      "learning_rate": 2.2081898201301186e-07,
      "logits/chosen": -2.400722026824951,
      "logits/rejected": -2.328423023223877,
      "logps/chosen": -251.57211303710938,
      "logps/rejected": -219.1768035888672,
      "loss": 0.5368,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.13836149871349335,
      "rewards/margins": 0.6494232416152954,
      "rewards/rejected": -0.78778475522995,
      "step": 1750
    },
    {
      "epoch": 1.82,
      "learning_rate": 2.1890547263681592e-07,
      "logits/chosen": -2.4296817779541016,
      "logits/rejected": -2.367189407348633,
      "logps/chosen": -271.3897399902344,
      "logps/rejected": -215.4512481689453,
      "loss": 0.5408,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.08417050540447235,
      "rewards/margins": 0.754088282585144,
      "rewards/rejected": -0.8382587432861328,
      "step": 1760
    },
    {
      "epoch": 1.83,
      "learning_rate": 2.1699196326061998e-07,
      "logits/chosen": -2.3716368675231934,
      "logits/rejected": -2.3670201301574707,
      "logps/chosen": -265.5472106933594,
      "logps/rejected": -241.4296875,
      "loss": 0.5596,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14254868030548096,
      "rewards/margins": 0.701485276222229,
      "rewards/rejected": -0.8440340161323547,
      "step": 1770
    },
    {
      "epoch": 1.84,
      "learning_rate": 2.1507845388442402e-07,
      "logits/chosen": -2.396054744720459,
      "logits/rejected": -2.285252332687378,
      "logps/chosen": -287.78558349609375,
      "logps/rejected": -233.22732543945312,
      "loss": 0.5295,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.057570360600948334,
      "rewards/margins": 0.7760987281799316,
      "rewards/rejected": -0.8336690664291382,
      "step": 1780
    },
    {
      "epoch": 1.85,
      "learning_rate": 2.1316494450822808e-07,
      "logits/chosen": -2.3319549560546875,
      "logits/rejected": -2.3298840522766113,
      "logps/chosen": -269.1609191894531,
      "logps/rejected": -242.9523468017578,
      "loss": 0.5454,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12093748152256012,
      "rewards/margins": 0.7203799486160278,
      "rewards/rejected": -0.8413174748420715,
      "step": 1790
    },
    {
      "epoch": 1.86,
      "learning_rate": 2.1125143513203214e-07,
      "logits/chosen": -2.4473745822906494,
      "logits/rejected": -2.4215996265411377,
      "logps/chosen": -266.4071960449219,
      "logps/rejected": -224.84756469726562,
      "loss": 0.5133,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.14717252552509308,
      "rewards/margins": 0.730695366859436,
      "rewards/rejected": -0.8778678774833679,
      "step": 1800
    },
    {
      "epoch": 1.87,
      "learning_rate": 2.093379257558362e-07,
      "logits/chosen": -2.426567316055298,
      "logits/rejected": -2.4301810264587402,
      "logps/chosen": -274.27569580078125,
      "logps/rejected": -237.6707305908203,
      "loss": 0.5553,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.19985240697860718,
      "rewards/margins": 0.6345885992050171,
      "rewards/rejected": -0.8344410061836243,
      "step": 1810
    },
    {
      "epoch": 1.88,
      "learning_rate": 2.0742441637964026e-07,
      "logits/chosen": -2.32918119430542,
      "logits/rejected": -2.315727710723877,
      "logps/chosen": -299.13201904296875,
      "logps/rejected": -238.65219116210938,
      "loss": 0.5101,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08389053493738174,
      "rewards/margins": 0.9151546359062195,
      "rewards/rejected": -0.999045193195343,
      "step": 1820
    },
    {
      "epoch": 1.89,
      "learning_rate": 2.055109070034443e-07,
      "logits/chosen": -2.4138271808624268,
      "logits/rejected": -2.395538091659546,
      "logps/chosen": -287.26788330078125,
      "logps/rejected": -256.14886474609375,
      "loss": 0.5232,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.10842672735452652,
      "rewards/margins": 0.7822098731994629,
      "rewards/rejected": -0.8906365633010864,
      "step": 1830
    },
    {
      "epoch": 1.9,
      "learning_rate": 2.0359739762724836e-07,
      "logits/chosen": -2.415120840072632,
      "logits/rejected": -2.359282970428467,
      "logps/chosen": -279.69464111328125,
      "logps/rejected": -237.9990234375,
      "loss": 0.5288,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.1128331646323204,
      "rewards/margins": 0.7950798869132996,
      "rewards/rejected": -0.9079130291938782,
      "step": 1840
    },
    {
      "epoch": 1.91,
      "learning_rate": 2.0168388825105242e-07,
      "logits/chosen": -2.342097520828247,
      "logits/rejected": -2.2842659950256348,
      "logps/chosen": -260.4648742675781,
      "logps/rejected": -264.97601318359375,
      "loss": 0.516,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.14475753903388977,
      "rewards/margins": 0.8012211918830872,
      "rewards/rejected": -0.9459787607192993,
      "step": 1850
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.997703788748565e-07,
      "logits/chosen": -2.3480544090270996,
      "logits/rejected": -2.300661325454712,
      "logps/chosen": -278.40704345703125,
      "logps/rejected": -228.9430694580078,
      "loss": 0.5331,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2291232794523239,
      "rewards/margins": 0.6820493936538696,
      "rewards/rejected": -0.9111726880073547,
      "step": 1860
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.9785686949866055e-07,
      "logits/chosen": -2.374687910079956,
      "logits/rejected": -2.449584484100342,
      "logps/chosen": -275.783447265625,
      "logps/rejected": -233.29403686523438,
      "loss": 0.5627,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1256851851940155,
      "rewards/margins": 0.5714830160140991,
      "rewards/rejected": -0.6971681714057922,
      "step": 1870
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.9594336012246458e-07,
      "logits/chosen": -2.4475274085998535,
      "logits/rejected": -2.3420703411102295,
      "logps/chosen": -270.1884460449219,
      "logps/rejected": -235.0321502685547,
      "loss": 0.5117,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.15421505272388458,
      "rewards/margins": 0.7101847529411316,
      "rewards/rejected": -0.864399790763855,
      "step": 1880
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.9402985074626865e-07,
      "logits/chosen": -2.379213333129883,
      "logits/rejected": -2.3949854373931885,
      "logps/chosen": -269.8036804199219,
      "logps/rejected": -228.36856079101562,
      "loss": 0.5536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1839611977338791,
      "rewards/margins": 0.684939980506897,
      "rewards/rejected": -0.8689011335372925,
      "step": 1890
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.921163413700727e-07,
      "logits/chosen": -2.3342185020446777,
      "logits/rejected": -2.291137218475342,
      "logps/chosen": -280.77459716796875,
      "logps/rejected": -209.2417755126953,
      "loss": 0.5782,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.22546765208244324,
      "rewards/margins": 0.6627296209335327,
      "rewards/rejected": -0.8881972432136536,
      "step": 1900
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.9020283199387677e-07,
      "logits/chosen": -2.4452133178710938,
      "logits/rejected": -2.3817238807678223,
      "logps/chosen": -262.40533447265625,
      "logps/rejected": -233.92431640625,
      "loss": 0.552,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.05952071025967598,
      "rewards/margins": 0.7161270976066589,
      "rewards/rejected": -0.7756478190422058,
      "step": 1910
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.8828932261768083e-07,
      "logits/chosen": -2.432097911834717,
      "logits/rejected": -2.338268995285034,
      "logps/chosen": -266.4609375,
      "logps/rejected": -244.36953735351562,
      "loss": 0.5319,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.1674240082502365,
      "rewards/margins": 0.6536375284194946,
      "rewards/rejected": -0.8210614919662476,
      "step": 1920
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.8637581324148487e-07,
      "logits/chosen": -2.4438672065734863,
      "logits/rejected": -2.4159255027770996,
      "logps/chosen": -265.9490051269531,
      "logps/rejected": -240.15011596679688,
      "loss": 0.5412,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11111648380756378,
      "rewards/margins": 0.8190618753433228,
      "rewards/rejected": -0.9301783442497253,
      "step": 1930
    },
    {
      "epoch": 2.0,
      "eval_logits/chosen": -2.0748343467712402,
      "eval_logits/rejected": -1.9509462118148804,
      "eval_logps/chosen": -266.15826416015625,
      "eval_logps/rejected": -227.92453002929688,
      "eval_loss": 0.5318232774734497,
      "eval_rewards/accuracies": 0.7419999837875366,
      "eval_rewards/chosen": -0.1497425138950348,
      "eval_rewards/margins": 0.7142792344093323,
      "eval_rewards/rejected": -0.8640217185020447,
      "eval_runtime": 456.7367,
      "eval_samples_per_second": 4.379,
      "eval_steps_per_second": 0.274,
      "step": 1936
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.8446230386528893e-07,
      "logits/chosen": -2.3549647331237793,
      "logits/rejected": -2.3090908527374268,
      "logps/chosen": -239.0998992919922,
      "logps/rejected": -226.9877166748047,
      "loss": 0.5444,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.196995347738266,
      "rewards/margins": 0.598027229309082,
      "rewards/rejected": -0.7950225472450256,
      "step": 1940
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.82548794489093e-07,
      "logits/chosen": -2.307945489883423,
      "logits/rejected": -2.3319668769836426,
      "logps/chosen": -243.7688446044922,
      "logps/rejected": -234.90756225585938,
      "loss": 0.5395,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.20119981467723846,
      "rewards/margins": 0.630052924156189,
      "rewards/rejected": -0.8312527537345886,
      "step": 1950
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.8063528511289706e-07,
      "logits/chosen": -2.358461380004883,
      "logits/rejected": -2.277463912963867,
      "logps/chosen": -286.9745178222656,
      "logps/rejected": -259.64605712890625,
      "loss": 0.5319,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10196640342473984,
      "rewards/margins": 0.7701470851898193,
      "rewards/rejected": -0.8721135258674622,
      "step": 1960
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.7872177573670112e-07,
      "logits/chosen": -2.4066734313964844,
      "logits/rejected": -2.4170022010803223,
      "logps/chosen": -252.1322479248047,
      "logps/rejected": -230.71755981445312,
      "loss": 0.5399,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.08831964433193207,
      "rewards/margins": 0.6736435890197754,
      "rewards/rejected": -0.7619632482528687,
      "step": 1970
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.7680826636050515e-07,
      "logits/chosen": -2.313403844833374,
      "logits/rejected": -2.280029773712158,
      "logps/chosen": -278.5584716796875,
      "logps/rejected": -234.1926727294922,
      "loss": 0.5248,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07849719375371933,
      "rewards/margins": 0.7140364050865173,
      "rewards/rejected": -0.7925336360931396,
      "step": 1980
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.7489475698430921e-07,
      "logits/chosen": -2.332839012145996,
      "logits/rejected": -2.3671531677246094,
      "logps/chosen": -249.85073852539062,
      "logps/rejected": -249.09408569335938,
      "loss": 0.5556,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.13992241024971008,
      "rewards/margins": 0.6826204061508179,
      "rewards/rejected": -0.8225427865982056,
      "step": 1990
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.7298124760811328e-07,
      "logits/chosen": -2.40509295463562,
      "logits/rejected": -2.385143756866455,
      "logps/chosen": -277.4422302246094,
      "logps/rejected": -224.10305786132812,
      "loss": 0.539,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1345512419939041,
      "rewards/margins": 0.709915280342102,
      "rewards/rejected": -0.8444665670394897,
      "step": 2000
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.7106773823191734e-07,
      "logits/chosen": -2.438018321990967,
      "logits/rejected": -2.372896909713745,
      "logps/chosen": -270.50103759765625,
      "logps/rejected": -240.79336547851562,
      "loss": 0.5304,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.14716067910194397,
      "rewards/margins": 0.8182382583618164,
      "rewards/rejected": -0.965398907661438,
      "step": 2010
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.691542288557214e-07,
      "logits/chosen": -2.41938853263855,
      "logits/rejected": -2.322054624557495,
      "logps/chosen": -297.6514892578125,
      "logps/rejected": -230.926513671875,
      "loss": 0.544,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17855019867420197,
      "rewards/margins": 0.7301106452941895,
      "rewards/rejected": -0.908660888671875,
      "step": 2020
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.6724071947952544e-07,
      "logits/chosen": -2.4344213008880615,
      "logits/rejected": -2.275275468826294,
      "logps/chosen": -240.5940704345703,
      "logps/rejected": -232.0761260986328,
      "loss": 0.5354,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1535104215145111,
      "rewards/margins": 0.6651692390441895,
      "rewards/rejected": -0.818679690361023,
      "step": 2030
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.653272101033295e-07,
      "logits/chosen": -2.4676742553710938,
      "logits/rejected": -2.3419291973114014,
      "logps/chosen": -276.0610656738281,
      "logps/rejected": -212.14321899414062,
      "loss": 0.5129,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.09980995208024979,
      "rewards/margins": 0.677939772605896,
      "rewards/rejected": -0.7777497172355652,
      "step": 2040
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.6341370072713356e-07,
      "logits/chosen": -2.471161365509033,
      "logits/rejected": -2.415436267852783,
      "logps/chosen": -302.96063232421875,
      "logps/rejected": -248.37313842773438,
      "loss": 0.5382,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.08729290962219238,
      "rewards/margins": 0.8466998934745789,
      "rewards/rejected": -0.9339929819107056,
      "step": 2050
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.6150019135093762e-07,
      "logits/chosen": -2.4522697925567627,
      "logits/rejected": -2.3724026679992676,
      "logps/chosen": -304.5672912597656,
      "logps/rejected": -232.2679443359375,
      "loss": 0.5288,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.1347544938325882,
      "rewards/margins": 0.7216586470603943,
      "rewards/rejected": -0.8564130663871765,
      "step": 2060
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.5958668197474169e-07,
      "logits/chosen": -2.4622063636779785,
      "logits/rejected": -2.3807263374328613,
      "logps/chosen": -272.76177978515625,
      "logps/rejected": -222.5310516357422,
      "loss": 0.5348,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.09426755458116531,
      "rewards/margins": 0.8474025726318359,
      "rewards/rejected": -0.9416700601577759,
      "step": 2070
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.5767317259854572e-07,
      "logits/chosen": -2.4014036655426025,
      "logits/rejected": -2.3767526149749756,
      "logps/chosen": -262.97088623046875,
      "logps/rejected": -233.16796875,
      "loss": 0.5329,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.1530064344406128,
      "rewards/margins": 0.7272775769233704,
      "rewards/rejected": -0.8802839517593384,
      "step": 2080
    },
    {
      "epoch": 2.16,
      "learning_rate": 1.5575966322234978e-07,
      "logits/chosen": -2.326734781265259,
      "logits/rejected": -2.2565207481384277,
      "logps/chosen": -259.03985595703125,
      "logps/rejected": -247.90286254882812,
      "loss": 0.5285,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.1133735179901123,
      "rewards/margins": 0.7463991641998291,
      "rewards/rejected": -0.8597726821899414,
      "step": 2090
    },
    {
      "epoch": 2.17,
      "learning_rate": 1.5384615384615385e-07,
      "logits/chosen": -2.378979444503784,
      "logits/rejected": -2.2936904430389404,
      "logps/chosen": -238.01724243164062,
      "logps/rejected": -218.4258575439453,
      "loss": 0.52,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.17722460627555847,
      "rewards/margins": 0.7926307320594788,
      "rewards/rejected": -0.9698553085327148,
      "step": 2100
    },
    {
      "epoch": 2.18,
      "learning_rate": 1.519326444699579e-07,
      "logits/chosen": -2.386594295501709,
      "logits/rejected": -2.346097469329834,
      "logps/chosen": -279.75958251953125,
      "logps/rejected": -252.79776000976562,
      "loss": 0.5138,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.10955911874771118,
      "rewards/margins": 0.7847574949264526,
      "rewards/rejected": -0.8943166732788086,
      "step": 2110
    },
    {
      "epoch": 2.19,
      "learning_rate": 1.5001913509376197e-07,
      "logits/chosen": -2.2668120861053467,
      "logits/rejected": -2.29984188079834,
      "logps/chosen": -242.16360473632812,
      "logps/rejected": -219.63143920898438,
      "loss": 0.5344,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.03612351417541504,
      "rewards/margins": 0.8138107061386108,
      "rewards/rejected": -0.8499342799186707,
      "step": 2120
    },
    {
      "epoch": 2.2,
      "learning_rate": 1.4810562571756603e-07,
      "logits/chosen": -2.405428647994995,
      "logits/rejected": -2.382072925567627,
      "logps/chosen": -260.3352355957031,
      "logps/rejected": -238.9024658203125,
      "loss": 0.5394,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1940733641386032,
      "rewards/margins": 0.6490978002548218,
      "rewards/rejected": -0.843171238899231,
      "step": 2130
    },
    {
      "epoch": 2.21,
      "learning_rate": 1.4619211634137007e-07,
      "logits/chosen": -2.417595624923706,
      "logits/rejected": -2.346921920776367,
      "logps/chosen": -296.86602783203125,
      "logps/rejected": -244.6204071044922,
      "loss": 0.5207,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.08298005163669586,
      "rewards/margins": 0.777125358581543,
      "rewards/rejected": -0.8601053357124329,
      "step": 2140
    },
    {
      "epoch": 2.22,
      "learning_rate": 1.4427860696517413e-07,
      "logits/chosen": -2.310767650604248,
      "logits/rejected": -2.2727725505828857,
      "logps/chosen": -276.6751708984375,
      "logps/rejected": -244.1461944580078,
      "loss": 0.5488,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.23952357470989227,
      "rewards/margins": 0.5698575973510742,
      "rewards/rejected": -0.8093811273574829,
      "step": 2150
    },
    {
      "epoch": 2.23,
      "learning_rate": 1.423650975889782e-07,
      "logits/chosen": -2.413959503173828,
      "logits/rejected": -2.317779302597046,
      "logps/chosen": -301.21697998046875,
      "logps/rejected": -230.67611694335938,
      "loss": 0.5223,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07718826830387115,
      "rewards/margins": 0.7954361438751221,
      "rewards/rejected": -0.872624397277832,
      "step": 2160
    },
    {
      "epoch": 2.24,
      "learning_rate": 1.4045158821278225e-07,
      "logits/chosen": -2.43796443939209,
      "logits/rejected": -2.3855981826782227,
      "logps/chosen": -281.7301025390625,
      "logps/rejected": -252.88900756835938,
      "loss": 0.5216,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07020233571529388,
      "rewards/margins": 0.8178524971008301,
      "rewards/rejected": -0.8880547285079956,
      "step": 2170
    },
    {
      "epoch": 2.25,
      "learning_rate": 1.3853807883658632e-07,
      "logits/chosen": -2.381850481033325,
      "logits/rejected": -2.3632988929748535,
      "logps/chosen": -286.48199462890625,
      "logps/rejected": -263.02325439453125,
      "loss": 0.5582,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06794221699237823,
      "rewards/margins": 0.675620436668396,
      "rewards/rejected": -0.743562638759613,
      "step": 2180
    },
    {
      "epoch": 2.26,
      "learning_rate": 1.3662456946039035e-07,
      "logits/chosen": -2.3416738510131836,
      "logits/rejected": -2.3051633834838867,
      "logps/chosen": -244.4949188232422,
      "logps/rejected": -206.51089477539062,
      "loss": 0.5339,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12611594796180725,
      "rewards/margins": 0.6276872158050537,
      "rewards/rejected": -0.7538031339645386,
      "step": 2190
    },
    {
      "epoch": 2.27,
      "learning_rate": 1.3471106008419441e-07,
      "logits/chosen": -2.3636796474456787,
      "logits/rejected": -2.3292782306671143,
      "logps/chosen": -273.59027099609375,
      "logps/rejected": -260.6876220703125,
      "loss": 0.5276,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.08774266391992569,
      "rewards/margins": 0.6979103088378906,
      "rewards/rejected": -0.7856529951095581,
      "step": 2200
    },
    {
      "epoch": 2.28,
      "learning_rate": 1.3279755070799848e-07,
      "logits/chosen": -2.3285207748413086,
      "logits/rejected": -2.2629952430725098,
      "logps/chosen": -284.96600341796875,
      "logps/rejected": -232.8168182373047,
      "loss": 0.5429,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10798223316669464,
      "rewards/margins": 0.7428350448608398,
      "rewards/rejected": -0.8508173227310181,
      "step": 2210
    },
    {
      "epoch": 2.29,
      "learning_rate": 1.3088404133180254e-07,
      "logits/chosen": -2.2993249893188477,
      "logits/rejected": -2.2543981075286865,
      "logps/chosen": -306.201171875,
      "logps/rejected": -226.76223754882812,
      "loss": 0.483,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06968189030885696,
      "rewards/margins": 0.9336912035942078,
      "rewards/rejected": -1.003373146057129,
      "step": 2220
    },
    {
      "epoch": 2.3,
      "learning_rate": 1.289705319556066e-07,
      "logits/chosen": -2.3141911029815674,
      "logits/rejected": -2.251255512237549,
      "logps/chosen": -252.2708282470703,
      "logps/rejected": -221.80953979492188,
      "loss": 0.5277,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.14586138725280762,
      "rewards/margins": 0.6129014492034912,
      "rewards/rejected": -0.7587628960609436,
      "step": 2230
    },
    {
      "epoch": 2.31,
      "learning_rate": 1.2705702257941064e-07,
      "logits/chosen": -2.373944044113159,
      "logits/rejected": -2.362863779067993,
      "logps/chosen": -257.61285400390625,
      "logps/rejected": -234.9366455078125,
      "loss": 0.5122,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1866065412759781,
      "rewards/margins": 0.6813262701034546,
      "rewards/rejected": -0.8679327964782715,
      "step": 2240
    },
    {
      "epoch": 2.32,
      "learning_rate": 1.251435132032147e-07,
      "logits/chosen": -2.3465607166290283,
      "logits/rejected": -2.338548421859741,
      "logps/chosen": -273.64178466796875,
      "logps/rejected": -259.8929748535156,
      "loss": 0.5306,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0799819678068161,
      "rewards/margins": 0.7552547454833984,
      "rewards/rejected": -0.8352367281913757,
      "step": 2250
    },
    {
      "epoch": 2.33,
      "learning_rate": 1.2323000382701873e-07,
      "logits/chosen": -2.3777191638946533,
      "logits/rejected": -2.3778316974639893,
      "logps/chosen": -270.16326904296875,
      "logps/rejected": -229.1251983642578,
      "loss": 0.5575,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2065315544605255,
      "rewards/margins": 0.6355454921722412,
      "rewards/rejected": -0.8420770764350891,
      "step": 2260
    },
    {
      "epoch": 2.34,
      "learning_rate": 1.213164944508228e-07,
      "logits/chosen": -2.4501662254333496,
      "logits/rejected": -2.3497557640075684,
      "logps/chosen": -266.2723083496094,
      "logps/rejected": -269.751708984375,
      "loss": 0.5113,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.2602607309818268,
      "rewards/margins": 0.6484657526016235,
      "rewards/rejected": -0.9087265133857727,
      "step": 2270
    },
    {
      "epoch": 2.35,
      "learning_rate": 1.1940298507462686e-07,
      "logits/chosen": -2.2452497482299805,
      "logits/rejected": -2.276188850402832,
      "logps/chosen": -261.48919677734375,
      "logps/rejected": -211.9011688232422,
      "loss": 0.5202,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0977492704987526,
      "rewards/margins": 0.8213205337524414,
      "rewards/rejected": -0.9190698862075806,
      "step": 2280
    },
    {
      "epoch": 2.37,
      "learning_rate": 1.1748947569843092e-07,
      "logits/chosen": -2.39143967628479,
      "logits/rejected": -2.307307481765747,
      "logps/chosen": -263.43524169921875,
      "logps/rejected": -225.5679473876953,
      "loss": 0.5088,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.12165951728820801,
      "rewards/margins": 0.8993788957595825,
      "rewards/rejected": -1.021038293838501,
      "step": 2290
    },
    {
      "epoch": 2.38,
      "learning_rate": 1.1557596632223497e-07,
      "logits/chosen": -2.361375331878662,
      "logits/rejected": -2.375532627105713,
      "logps/chosen": -272.00408935546875,
      "logps/rejected": -224.09286499023438,
      "loss": 0.5223,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.13877837359905243,
      "rewards/margins": 0.8421686291694641,
      "rewards/rejected": -0.9809468984603882,
      "step": 2300
    },
    {
      "epoch": 2.39,
      "learning_rate": 1.1366245694603903e-07,
      "logits/chosen": -2.4291224479675293,
      "logits/rejected": -2.245253801345825,
      "logps/chosen": -265.1824645996094,
      "logps/rejected": -249.3574676513672,
      "loss": 0.5579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.24909143149852753,
      "rewards/margins": 0.6685684323310852,
      "rewards/rejected": -0.9176598787307739,
      "step": 2310
    },
    {
      "epoch": 2.4,
      "learning_rate": 1.1174894756984308e-07,
      "logits/chosen": -2.2975034713745117,
      "logits/rejected": -2.394157886505127,
      "logps/chosen": -254.5048370361328,
      "logps/rejected": -224.20376586914062,
      "loss": 0.5215,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.18976199626922607,
      "rewards/margins": 0.8317477107048035,
      "rewards/rejected": -1.0215097665786743,
      "step": 2320
    },
    {
      "epoch": 2.41,
      "learning_rate": 1.0983543819364714e-07,
      "logits/chosen": -2.4635863304138184,
      "logits/rejected": -2.4494881629943848,
      "logps/chosen": -278.895263671875,
      "logps/rejected": -231.98135375976562,
      "loss": 0.4772,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.08000790327787399,
      "rewards/margins": 0.9092146158218384,
      "rewards/rejected": -0.989222526550293,
      "step": 2330
    },
    {
      "epoch": 2.42,
      "learning_rate": 1.079219288174512e-07,
      "logits/chosen": -2.3342247009277344,
      "logits/rejected": -2.319608211517334,
      "logps/chosen": -247.49368286132812,
      "logps/rejected": -209.24111938476562,
      "loss": 0.5371,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.20573286712169647,
      "rewards/margins": 0.5834436416625977,
      "rewards/rejected": -0.7891765236854553,
      "step": 2340
    },
    {
      "epoch": 2.43,
      "learning_rate": 1.0600841944125525e-07,
      "logits/chosen": -2.436828136444092,
      "logits/rejected": -2.3965752124786377,
      "logps/chosen": -271.6636047363281,
      "logps/rejected": -221.7318572998047,
      "loss": 0.5528,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.20491096377372742,
      "rewards/margins": 0.6560606360435486,
      "rewards/rejected": -0.8609716296195984,
      "step": 2350
    },
    {
      "epoch": 2.44,
      "learning_rate": 1.0409491006505931e-07,
      "logits/chosen": -2.400538682937622,
      "logits/rejected": -2.284837484359741,
      "logps/chosen": -283.9150390625,
      "logps/rejected": -229.0013885498047,
      "loss": 0.5457,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.1943405717611313,
      "rewards/margins": 0.7493225336074829,
      "rewards/rejected": -0.9436630010604858,
      "step": 2360
    },
    {
      "epoch": 2.45,
      "learning_rate": 1.0218140068886336e-07,
      "logits/chosen": -2.377981662750244,
      "logits/rejected": -2.2820467948913574,
      "logps/chosen": -260.76312255859375,
      "logps/rejected": -221.4712371826172,
      "loss": 0.5534,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.21287551522254944,
      "rewards/margins": 0.5487321615219116,
      "rewards/rejected": -0.7616077661514282,
      "step": 2370
    },
    {
      "epoch": 2.46,
      "learning_rate": 1.0026789131266743e-07,
      "logits/chosen": -2.295510768890381,
      "logits/rejected": -2.329360246658325,
      "logps/chosen": -252.51107788085938,
      "logps/rejected": -242.34616088867188,
      "loss": 0.5429,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1736653447151184,
      "rewards/margins": 0.6323011517524719,
      "rewards/rejected": -0.8059664964675903,
      "step": 2380
    },
    {
      "epoch": 2.47,
      "learning_rate": 9.835438193647149e-08,
      "logits/chosen": -2.3245301246643066,
      "logits/rejected": -2.372647523880005,
      "logps/chosen": -255.889892578125,
      "logps/rejected": -230.44961547851562,
      "loss": 0.5579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.15318112075328827,
      "rewards/margins": 0.6755190491676331,
      "rewards/rejected": -0.8287001848220825,
      "step": 2390
    },
    {
      "epoch": 2.48,
      "learning_rate": 9.644087256027554e-08,
      "logits/chosen": -2.393808364868164,
      "logits/rejected": -2.387742280960083,
      "logps/chosen": -239.28219604492188,
      "logps/rejected": -218.7506103515625,
      "loss": 0.536,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.13704940676689148,
      "rewards/margins": 0.6621817946434021,
      "rewards/rejected": -0.7992312908172607,
      "step": 2400
    },
    {
      "epoch": 2.49,
      "learning_rate": 9.45273631840796e-08,
      "logits/chosen": -2.3953936100006104,
      "logits/rejected": -2.4050917625427246,
      "logps/chosen": -278.6383361816406,
      "logps/rejected": -231.466552734375,
      "loss": 0.5329,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07729502022266388,
      "rewards/margins": 0.736717700958252,
      "rewards/rejected": -0.8140127062797546,
      "step": 2410
    },
    {
      "epoch": 2.5,
      "learning_rate": 9.261385380788366e-08,
      "logits/chosen": -2.276078939437866,
      "logits/rejected": -2.2849624156951904,
      "logps/chosen": -239.13525390625,
      "logps/rejected": -222.2736053466797,
      "loss": 0.5249,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.2353070229291916,
      "rewards/margins": 0.6328269243240356,
      "rewards/rejected": -0.8681339025497437,
      "step": 2420
    },
    {
      "epoch": 2.51,
      "learning_rate": 9.070034443168771e-08,
      "logits/chosen": -2.387070894241333,
      "logits/rejected": -2.37086820602417,
      "logps/chosen": -261.84112548828125,
      "logps/rejected": -227.5956268310547,
      "loss": 0.557,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.2481728047132492,
      "rewards/margins": 0.6492807865142822,
      "rewards/rejected": -0.8974536657333374,
      "step": 2430
    },
    {
      "epoch": 2.52,
      "learning_rate": 8.878683505549177e-08,
      "logits/chosen": -2.2902541160583496,
      "logits/rejected": -2.3299739360809326,
      "logps/chosen": -267.1860656738281,
      "logps/rejected": -226.9349822998047,
      "loss": 0.5107,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.10977288335561752,
      "rewards/margins": 0.7155038714408875,
      "rewards/rejected": -0.8252767324447632,
      "step": 2440
    },
    {
      "epoch": 2.53,
      "learning_rate": 8.687332567929582e-08,
      "logits/chosen": -2.3206660747528076,
      "logits/rejected": -2.3494954109191895,
      "logps/chosen": -291.82305908203125,
      "logps/rejected": -246.83761596679688,
      "loss": 0.5355,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.044042427092790604,
      "rewards/margins": 0.900373637676239,
      "rewards/rejected": -0.9444161653518677,
      "step": 2450
    },
    {
      "epoch": 2.54,
      "learning_rate": 8.495981630309988e-08,
      "logits/chosen": -2.2711682319641113,
      "logits/rejected": -2.290236473083496,
      "logps/chosen": -300.72979736328125,
      "logps/rejected": -218.26022338867188,
      "loss": 0.5448,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1306503415107727,
      "rewards/margins": 0.5897533297538757,
      "rewards/rejected": -0.7204037308692932,
      "step": 2460
    },
    {
      "epoch": 2.55,
      "learning_rate": 8.304630692690395e-08,
      "logits/chosen": -2.317441463470459,
      "logits/rejected": -2.2715091705322266,
      "logps/chosen": -262.0405578613281,
      "logps/rejected": -205.5135955810547,
      "loss": 0.5286,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.14162805676460266,
      "rewards/margins": 0.7206326723098755,
      "rewards/rejected": -0.8622606992721558,
      "step": 2470
    },
    {
      "epoch": 2.56,
      "learning_rate": 8.1132797550708e-08,
      "logits/chosen": -2.3969407081604004,
      "logits/rejected": -2.3611950874328613,
      "logps/chosen": -267.4500427246094,
      "logps/rejected": -221.7566375732422,
      "loss": 0.5277,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.15396210551261902,
      "rewards/margins": 0.7070397138595581,
      "rewards/rejected": -0.8610017895698547,
      "step": 2480
    },
    {
      "epoch": 2.57,
      "learning_rate": 7.921928817451206e-08,
      "logits/chosen": -2.4169304370880127,
      "logits/rejected": -2.321960926055908,
      "logps/chosen": -277.09588623046875,
      "logps/rejected": -227.67959594726562,
      "loss": 0.5088,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.17860722541809082,
      "rewards/margins": 0.7987648248672485,
      "rewards/rejected": -0.9773720502853394,
      "step": 2490
    },
    {
      "epoch": 2.58,
      "learning_rate": 7.73057787983161e-08,
      "logits/chosen": -2.3953964710235596,
      "logits/rejected": -2.301238536834717,
      "logps/chosen": -292.7876892089844,
      "logps/rejected": -266.57476806640625,
      "loss": 0.5165,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08117020130157471,
      "rewards/margins": 0.7519367933273315,
      "rewards/rejected": -0.8331069946289062,
      "step": 2500
    },
    {
      "epoch": 2.59,
      "learning_rate": 7.539226942212017e-08,
      "logits/chosen": -2.3565969467163086,
      "logits/rejected": -2.3130733966827393,
      "logps/chosen": -243.97903442382812,
      "logps/rejected": -219.99301147460938,
      "loss": 0.5185,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.11240849643945694,
      "rewards/margins": 0.8732815980911255,
      "rewards/rejected": -0.9856899976730347,
      "step": 2510
    },
    {
      "epoch": 2.6,
      "learning_rate": 7.347876004592423e-08,
      "logits/chosen": -2.4705801010131836,
      "logits/rejected": -2.40214467048645,
      "logps/chosen": -280.0275573730469,
      "logps/rejected": -232.6969451904297,
      "loss": 0.5403,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.14689256250858307,
      "rewards/margins": 0.7363706231117249,
      "rewards/rejected": -0.883263111114502,
      "step": 2520
    },
    {
      "epoch": 2.61,
      "learning_rate": 7.156525066972828e-08,
      "logits/chosen": -2.299910068511963,
      "logits/rejected": -2.3627266883850098,
      "logps/chosen": -242.22579956054688,
      "logps/rejected": -227.53482055664062,
      "loss": 0.5402,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.088679239153862,
      "rewards/margins": 0.6972663402557373,
      "rewards/rejected": -0.7859456539154053,
      "step": 2530
    },
    {
      "epoch": 2.62,
      "learning_rate": 6.965174129353234e-08,
      "logits/chosen": -2.3531718254089355,
      "logits/rejected": -2.2922921180725098,
      "logps/chosen": -302.03350830078125,
      "logps/rejected": -258.0994567871094,
      "loss": 0.5287,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.10454048961400986,
      "rewards/margins": 0.8333840370178223,
      "rewards/rejected": -0.9379245042800903,
      "step": 2540
    },
    {
      "epoch": 2.63,
      "learning_rate": 6.773823191733639e-08,
      "logits/chosen": -2.284903049468994,
      "logits/rejected": -2.2148680686950684,
      "logps/chosen": -259.69036865234375,
      "logps/rejected": -222.1662139892578,
      "loss": 0.5297,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.24571335315704346,
      "rewards/margins": 0.7015795707702637,
      "rewards/rejected": -0.9472929239273071,
      "step": 2550
    },
    {
      "epoch": 2.64,
      "learning_rate": 6.582472254114045e-08,
      "logits/chosen": -2.3052189350128174,
      "logits/rejected": -2.3175923824310303,
      "logps/chosen": -252.48291015625,
      "logps/rejected": -223.8980712890625,
      "loss": 0.5649,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12469018995761871,
      "rewards/margins": 0.6323705911636353,
      "rewards/rejected": -0.757060706615448,
      "step": 2560
    },
    {
      "epoch": 2.65,
      "learning_rate": 6.391121316494451e-08,
      "logits/chosen": -2.3261055946350098,
      "logits/rejected": -2.3110244274139404,
      "logps/chosen": -295.005859375,
      "logps/rejected": -239.6460418701172,
      "loss": 0.5248,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.10888131707906723,
      "rewards/margins": 0.8443535566329956,
      "rewards/rejected": -0.953234851360321,
      "step": 2570
    },
    {
      "epoch": 2.66,
      "learning_rate": 6.199770378874856e-08,
      "logits/chosen": -2.377129077911377,
      "logits/rejected": -2.2884180545806885,
      "logps/chosen": -272.9131164550781,
      "logps/rejected": -225.4416961669922,
      "loss": 0.5302,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07070966064929962,
      "rewards/margins": 0.9023112058639526,
      "rewards/rejected": -0.9730209112167358,
      "step": 2580
    },
    {
      "epoch": 2.67,
      "learning_rate": 6.008419441255262e-08,
      "logits/chosen": -2.3185131549835205,
      "logits/rejected": -2.260045051574707,
      "logps/chosen": -299.19635009765625,
      "logps/rejected": -217.824951171875,
      "loss": 0.5244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07015608996152878,
      "rewards/margins": 0.8233577013015747,
      "rewards/rejected": -0.8935137987136841,
      "step": 2590
    },
    {
      "epoch": 2.69,
      "learning_rate": 5.817068503635668e-08,
      "logits/chosen": -2.4003567695617676,
      "logits/rejected": -2.3623907566070557,
      "logps/chosen": -288.9065856933594,
      "logps/rejected": -222.6360626220703,
      "loss": 0.515,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.12081106007099152,
      "rewards/margins": 0.8377307057380676,
      "rewards/rejected": -0.958541750907898,
      "step": 2600
    },
    {
      "epoch": 2.7,
      "learning_rate": 5.6257175660160735e-08,
      "logits/chosen": -2.420441150665283,
      "logits/rejected": -2.37391996383667,
      "logps/chosen": -295.19952392578125,
      "logps/rejected": -238.8549041748047,
      "loss": 0.516,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18193045258522034,
      "rewards/margins": 0.7073780298233032,
      "rewards/rejected": -0.8893085718154907,
      "step": 2610
    },
    {
      "epoch": 2.71,
      "learning_rate": 5.4343666283964784e-08,
      "logits/chosen": -2.4277868270874023,
      "logits/rejected": -2.3764290809631348,
      "logps/chosen": -263.9327697753906,
      "logps/rejected": -249.3427734375,
      "loss": 0.5214,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07714501768350601,
      "rewards/margins": 0.7166513204574585,
      "rewards/rejected": -0.7937964200973511,
      "step": 2620
    },
    {
      "epoch": 2.72,
      "learning_rate": 5.243015690776884e-08,
      "logits/chosen": -2.467040538787842,
      "logits/rejected": -2.406569719314575,
      "logps/chosen": -285.0174865722656,
      "logps/rejected": -235.902587890625,
      "loss": 0.5146,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.15382210910320282,
      "rewards/margins": 0.7074260711669922,
      "rewards/rejected": -0.8612481951713562,
      "step": 2630
    },
    {
      "epoch": 2.73,
      "learning_rate": 5.05166475315729e-08,
      "logits/chosen": -2.3759026527404785,
      "logits/rejected": -2.323474884033203,
      "logps/chosen": -263.00689697265625,
      "logps/rejected": -218.7880859375,
      "loss": 0.5388,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.2826555371284485,
      "rewards/margins": 0.6195917129516602,
      "rewards/rejected": -0.9022472500801086,
      "step": 2640
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.860313815537696e-08,
      "logits/chosen": -2.4053902626037598,
      "logits/rejected": -2.36479115486145,
      "logps/chosen": -284.4691467285156,
      "logps/rejected": -266.75579833984375,
      "loss": 0.5364,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11270469427108765,
      "rewards/margins": 0.7914165258407593,
      "rewards/rejected": -0.9041212201118469,
      "step": 2650
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.668962877918101e-08,
      "logits/chosen": -2.380582332611084,
      "logits/rejected": -2.2785563468933105,
      "logps/chosen": -288.0699768066406,
      "logps/rejected": -278.1865234375,
      "loss": 0.5236,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12235669791698456,
      "rewards/margins": 0.8113776445388794,
      "rewards/rejected": -0.9337342381477356,
      "step": 2660
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.477611940298507e-08,
      "logits/chosen": -2.4007792472839355,
      "logits/rejected": -2.3088767528533936,
      "logps/chosen": -245.1487274169922,
      "logps/rejected": -227.521728515625,
      "loss": 0.4938,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.13851900398731232,
      "rewards/margins": 0.8027512431144714,
      "rewards/rejected": -0.9412702322006226,
      "step": 2670
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.2862610026789124e-08,
      "logits/chosen": -2.4124245643615723,
      "logits/rejected": -2.296288013458252,
      "logps/chosen": -259.74029541015625,
      "logps/rejected": -223.14627075195312,
      "loss": 0.5476,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1544261872768402,
      "rewards/margins": 0.7074520587921143,
      "rewards/rejected": -0.8618782162666321,
      "step": 2680
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.0949100650593186e-08,
      "logits/chosen": -2.3861641883850098,
      "logits/rejected": -2.328504800796509,
      "logps/chosen": -256.6800231933594,
      "logps/rejected": -243.67514038085938,
      "loss": 0.546,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.25118106603622437,
      "rewards/margins": 0.5502891540527344,
      "rewards/rejected": -0.8014701008796692,
      "step": 2690
    },
    {
      "epoch": 2.79,
      "learning_rate": 3.903559127439724e-08,
      "logits/chosen": -2.294541358947754,
      "logits/rejected": -2.2779922485351562,
      "logps/chosen": -273.6397399902344,
      "logps/rejected": -230.7729949951172,
      "loss": 0.5064,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.11550472676753998,
      "rewards/margins": 0.7522660493850708,
      "rewards/rejected": -0.867770791053772,
      "step": 2700
    },
    {
      "epoch": 2.8,
      "learning_rate": 3.71220818982013e-08,
      "logits/chosen": -2.406259536743164,
      "logits/rejected": -2.3714747428894043,
      "logps/chosen": -273.5616760253906,
      "logps/rejected": -218.18923950195312,
      "loss": 0.5481,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.23518535494804382,
      "rewards/margins": 0.6195680499076843,
      "rewards/rejected": -0.8547533750534058,
      "step": 2710
    },
    {
      "epoch": 2.81,
      "learning_rate": 3.520857252200535e-08,
      "logits/chosen": -2.363008975982666,
      "logits/rejected": -2.3325300216674805,
      "logps/chosen": -300.9599609375,
      "logps/rejected": -242.6225128173828,
      "loss": 0.5323,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.039127424359321594,
      "rewards/margins": 0.9830048680305481,
      "rewards/rejected": -1.022132158279419,
      "step": 2720
    },
    {
      "epoch": 2.82,
      "learning_rate": 3.3295063145809414e-08,
      "logits/chosen": -2.280790328979492,
      "logits/rejected": -2.225207805633545,
      "logps/chosen": -266.9983825683594,
      "logps/rejected": -244.36279296875,
      "loss": 0.5561,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.14134183526039124,
      "rewards/margins": 0.7063352465629578,
      "rewards/rejected": -0.8476771116256714,
      "step": 2730
    },
    {
      "epoch": 2.83,
      "learning_rate": 3.138155376961347e-08,
      "logits/chosen": -2.288053512573242,
      "logits/rejected": -2.3323187828063965,
      "logps/chosen": -234.13241577148438,
      "logps/rejected": -226.0674285888672,
      "loss": 0.5667,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.14560513198375702,
      "rewards/margins": 0.7158702611923218,
      "rewards/rejected": -0.8614753484725952,
      "step": 2740
    },
    {
      "epoch": 2.84,
      "learning_rate": 2.9468044393417525e-08,
      "logits/chosen": -2.4299161434173584,
      "logits/rejected": -2.40264892578125,
      "logps/chosen": -273.2800598144531,
      "logps/rejected": -262.6374206542969,
      "loss": 0.5196,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06616147607564926,
      "rewards/margins": 0.8120719194412231,
      "rewards/rejected": -0.8782333135604858,
      "step": 2750
    },
    {
      "epoch": 2.85,
      "learning_rate": 2.755453501722158e-08,
      "logits/chosen": -2.37566876411438,
      "logits/rejected": -2.36662220954895,
      "logps/chosen": -285.37725830078125,
      "logps/rejected": -228.5401611328125,
      "loss": 0.5099,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09863577038049698,
      "rewards/margins": 0.8224757313728333,
      "rewards/rejected": -0.9211114645004272,
      "step": 2760
    },
    {
      "epoch": 2.86,
      "learning_rate": 2.564102564102564e-08,
      "logits/chosen": -2.399296283721924,
      "logits/rejected": -2.3836464881896973,
      "logps/chosen": -258.1005554199219,
      "logps/rejected": -225.07723999023438,
      "loss": 0.524,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.18826046586036682,
      "rewards/margins": 0.6354348659515381,
      "rewards/rejected": -0.8236953020095825,
      "step": 2770
    },
    {
      "epoch": 2.87,
      "learning_rate": 2.3727516264829695e-08,
      "logits/chosen": -2.320814609527588,
      "logits/rejected": -2.295502185821533,
      "logps/chosen": -238.77200317382812,
      "logps/rejected": -234.020263671875,
      "loss": 0.5344,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1772747039794922,
      "rewards/margins": 0.5961662530899048,
      "rewards/rejected": -0.7734408974647522,
      "step": 2780
    },
    {
      "epoch": 2.88,
      "learning_rate": 2.1814006888633754e-08,
      "logits/chosen": -2.334709405899048,
      "logits/rejected": -2.3521342277526855,
      "logps/chosen": -263.27349853515625,
      "logps/rejected": -233.0219268798828,
      "loss": 0.5471,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.2036319077014923,
      "rewards/margins": 0.6492605805397034,
      "rewards/rejected": -0.8528925180435181,
      "step": 2790
    },
    {
      "epoch": 2.89,
      "learning_rate": 1.990049751243781e-08,
      "logits/chosen": -2.3961801528930664,
      "logits/rejected": -2.344447612762451,
      "logps/chosen": -285.4869079589844,
      "logps/rejected": -231.99276733398438,
      "loss": 0.5319,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.11304112523794174,
      "rewards/margins": 0.6869500875473022,
      "rewards/rejected": -0.7999913096427917,
      "step": 2800
    },
    {
      "epoch": 2.9,
      "learning_rate": 1.7986988136241865e-08,
      "logits/chosen": -2.4139771461486816,
      "logits/rejected": -2.357755661010742,
      "logps/chosen": -262.9169006347656,
      "logps/rejected": -225.28134155273438,
      "loss": 0.4933,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.029026124626398087,
      "rewards/margins": 0.874340832233429,
      "rewards/rejected": -0.9033668637275696,
      "step": 2810
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.6073478760045924e-08,
      "logits/chosen": -2.2996811866760254,
      "logits/rejected": -2.224923849105835,
      "logps/chosen": -285.3755798339844,
      "logps/rejected": -201.12803649902344,
      "loss": 0.5432,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.24773137271404266,
      "rewards/margins": 0.5589512586593628,
      "rewards/rejected": -0.8066825866699219,
      "step": 2820
    },
    {
      "epoch": 2.92,
      "learning_rate": 1.4159969383849981e-08,
      "logits/chosen": -2.493635654449463,
      "logits/rejected": -2.4172415733337402,
      "logps/chosen": -312.3724670410156,
      "logps/rejected": -248.0670623779297,
      "loss": 0.5265,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.10385121405124664,
      "rewards/margins": 0.8151585459709167,
      "rewards/rejected": -0.9190096855163574,
      "step": 2830
    },
    {
      "epoch": 2.93,
      "learning_rate": 1.2246460007654037e-08,
      "logits/chosen": -2.365940570831299,
      "logits/rejected": -2.3057992458343506,
      "logps/chosen": -266.408447265625,
      "logps/rejected": -227.32492065429688,
      "loss": 0.571,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.23473727703094482,
      "rewards/margins": 0.5884072184562683,
      "rewards/rejected": -0.8231445550918579,
      "step": 2840
    },
    {
      "epoch": 2.94,
      "learning_rate": 1.0332950631458094e-08,
      "logits/chosen": -2.306847095489502,
      "logits/rejected": -2.352964401245117,
      "logps/chosen": -273.6145935058594,
      "logps/rejected": -230.6118927001953,
      "loss": 0.5353,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1490267664194107,
      "rewards/margins": 0.7372574210166931,
      "rewards/rejected": -0.8862841725349426,
      "step": 2850
    },
    {
      "epoch": 2.95,
      "learning_rate": 8.419441255262151e-09,
      "logits/chosen": -2.2987818717956543,
      "logits/rejected": -2.319676160812378,
      "logps/chosen": -267.0763854980469,
      "logps/rejected": -219.8603973388672,
      "loss": 0.5045,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.09555543214082718,
      "rewards/margins": 0.6565554738044739,
      "rewards/rejected": -0.7521108388900757,
      "step": 2860
    },
    {
      "epoch": 2.96,
      "learning_rate": 6.505931879066207e-09,
      "logits/chosen": -2.403472900390625,
      "logits/rejected": -2.3466238975524902,
      "logps/chosen": -235.06271362304688,
      "logps/rejected": -236.23068237304688,
      "loss": 0.5334,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.24678435921669006,
      "rewards/margins": 0.6536287665367126,
      "rewards/rejected": -0.9004132151603699,
      "step": 2870
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.592422502870264e-09,
      "logits/chosen": -2.413404941558838,
      "logits/rejected": -2.317659854888916,
      "logps/chosen": -259.46697998046875,
      "logps/rejected": -227.43246459960938,
      "loss": 0.5463,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.05970311164855957,
      "rewards/margins": 0.6795765161514282,
      "rewards/rejected": -0.7392796277999878,
      "step": 2880
    },
    {
      "epoch": 2.98,
      "learning_rate": 2.6789131266743202e-09,
      "logits/chosen": -2.3422017097473145,
      "logits/rejected": -2.2731831073760986,
      "logps/chosen": -233.3965301513672,
      "logps/rejected": -225.33566284179688,
      "loss": 0.5157,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2789511978626251,
      "rewards/margins": 0.6096433401107788,
      "rewards/rejected": -0.8885944485664368,
      "step": 2890
    },
    {
      "epoch": 3.0,
      "learning_rate": 7.654037504783773e-10,
      "logits/chosen": -2.3140885829925537,
      "logits/rejected": -2.351783275604248,
      "logps/chosen": -238.41293334960938,
      "logps/rejected": -245.66201782226562,
      "loss": 0.5454,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.1346302330493927,
      "rewards/margins": 0.7680613398551941,
      "rewards/rejected": -0.9026915431022644,
      "step": 2900
    },
    {
      "epoch": 3.0,
      "eval_logits/chosen": -2.0658671855926514,
      "eval_logits/rejected": -1.9411793947219849,
      "eval_logps/chosen": -266.13037109375,
      "eval_logps/rejected": -228.2694091796875,
      "eval_loss": 0.5265706777572632,
      "eval_rewards/accuracies": 0.7459999918937683,
      "eval_rewards/chosen": -0.1469534933567047,
      "eval_rewards/margins": 0.7515553832054138,
      "eval_rewards/rejected": -0.8985088467597961,
      "eval_runtime": 452.2504,
      "eval_samples_per_second": 4.422,
      "eval_steps_per_second": 0.276,
      "step": 2904
    },
    {
      "epoch": 3.0,
      "step": 2904,
      "total_flos": 0.0,
      "train_loss": 0.5642705829018732,
      "train_runtime": 71197.0849,
      "train_samples_per_second": 2.611,
      "train_steps_per_second": 0.041
    }
  ],
  "logging_steps": 10,
  "max_steps": 2904,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}