{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.2731018246027075,
  "eval_steps": 50,
  "global_step": 1450,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.015695507161075144,
      "grad_norm": 0.07061820477247238,
      "learning_rate": 4.999451708687114e-06,
      "logits/chosen": 15.001790046691895,
      "logits/rejected": 14.624488830566406,
      "logps/chosen": -0.33085882663726807,
      "logps/rejected": -0.24924471974372864,
      "loss": 1.0519,
      "rewards/accuracies": 0.23749999701976776,
      "rewards/chosen": -0.4962882399559021,
      "rewards/margins": -0.12242116779088974,
      "rewards/rejected": -0.37386709451675415,
      "step": 10
    },
    {
      "epoch": 0.03139101432215029,
      "grad_norm": 0.060370542109012604,
      "learning_rate": 4.997807075247147e-06,
      "logits/chosen": 15.272351264953613,
      "logits/rejected": 14.801017761230469,
      "logps/chosen": -0.3379867672920227,
      "logps/rejected": -0.24759705364704132,
      "loss": 1.0494,
      "rewards/accuracies": 0.15000000596046448,
      "rewards/chosen": -0.5069801211357117,
      "rewards/margins": -0.13558456301689148,
      "rewards/rejected": -0.3713955581188202,
      "step": 20
    },
    {
      "epoch": 0.047086521483225424,
      "grad_norm": 0.050332240760326385,
      "learning_rate": 4.9950668210706795e-06,
      "logits/chosen": 15.569372177124023,
      "logits/rejected": 15.249380111694336,
      "logps/chosen": -0.3347483277320862,
      "logps/rejected": -0.275604784488678,
      "loss": 1.0375,
      "rewards/accuracies": 0.21250000596046448,
      "rewards/chosen": -0.5021225214004517,
      "rewards/margins": -0.08871528506278992,
      "rewards/rejected": -0.41340717673301697,
      "step": 30
    },
    {
      "epoch": 0.06278202864430057,
      "grad_norm": 0.054660990834236145,
      "learning_rate": 4.9912321481237616e-06,
      "logits/chosen": 15.344067573547363,
      "logits/rejected": 14.881669998168945,
      "logps/chosen": -0.3336474299430847,
      "logps/rejected": -0.27252259850502014,
      "loss": 1.0375,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -0.5004712343215942,
      "rewards/margins": -0.09168727695941925,
      "rewards/rejected": -0.40878385305404663,
      "step": 40
    },
    {
      "epoch": 0.07847753580537571,
      "grad_norm": 0.060405269265174866,
      "learning_rate": 4.986304738420684e-06,
      "logits/chosen": 15.5701904296875,
      "logits/rejected": 15.60998821258545,
      "logps/chosen": -0.3075069785118103,
      "logps/rejected": -0.24850216507911682,
      "loss": 1.0341,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -0.46126049757003784,
      "rewards/margins": -0.08850721269845963,
      "rewards/rejected": -0.3727532923221588,
      "step": 50
    },
    {
      "epoch": 0.07847753580537571,
      "eval_logits/chosen": 15.719181060791016,
      "eval_logits/rejected": 15.20205307006836,
      "eval_logps/chosen": -0.32490479946136475,
      "eval_logps/rejected": -0.26058924198150635,
      "eval_loss": 1.0295902490615845,
      "eval_rewards/accuracies": 0.26923078298568726,
      "eval_rewards/chosen": -0.48735716938972473,
      "eval_rewards/margins": -0.09647335112094879,
      "eval_rewards/rejected": -0.39088380336761475,
      "eval_runtime": 14.6102,
      "eval_samples_per_second": 28.199,
      "eval_steps_per_second": 3.559,
      "step": 50
    },
    {
      "epoch": 0.09417304296645085,
      "grad_norm": 0.06511708348989487,
      "learning_rate": 4.980286753286196e-06,
      "logits/chosen": 15.571348190307617,
      "logits/rejected": 15.418828010559082,
      "logps/chosen": -0.3217025101184845,
      "logps/rejected": -0.2813836932182312,
      "loss": 1.0385,
      "rewards/accuracies": 0.23749999701976776,
      "rewards/chosen": -0.48255378007888794,
      "rewards/margins": -0.060478221625089645,
      "rewards/rejected": -0.4220755696296692,
      "step": 60
    },
    {
      "epoch": 0.109868550127526,
      "grad_norm": 0.06922140717506409,
      "learning_rate": 4.973180832407471e-06,
      "logits/chosen": 15.926614761352539,
      "logits/rejected": 15.90565299987793,
      "logps/chosen": -0.34742942452430725,
      "logps/rejected": -0.26853513717651367,
      "loss": 1.0417,
      "rewards/accuracies": 0.21250000596046448,
      "rewards/chosen": -0.5211440324783325,
      "rewards/margins": -0.11834144592285156,
      "rewards/rejected": -0.40280264616012573,
      "step": 70
    },
    {
      "epoch": 0.12556405728860115,
      "grad_norm": 0.07853339612483978,
      "learning_rate": 4.964990092676263e-06,
      "logits/chosen": 15.876957893371582,
      "logits/rejected": 15.636863708496094,
      "logps/chosen": -0.34999752044677734,
      "logps/rejected": -0.2652502655982971,
      "loss": 1.0255,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.524996280670166,
      "rewards/margins": -0.12712089717388153,
      "rewards/rejected": -0.3978753685951233,
      "step": 80
    },
    {
      "epoch": 0.14125956444967627,
      "grad_norm": 0.07262148708105087,
      "learning_rate": 4.9557181268217225e-06,
      "logits/chosen": 16.037456512451172,
      "logits/rejected": 15.742408752441406,
      "logps/chosen": -0.32482510805130005,
      "logps/rejected": -0.2520079016685486,
      "loss": 1.0297,
      "rewards/accuracies": 0.21250000596046448,
      "rewards/chosen": -0.48723769187927246,
      "rewards/margins": -0.1092257872223854,
      "rewards/rejected": -0.3780118525028229,
      "step": 90
    },
    {
      "epoch": 0.15695507161075142,
      "grad_norm": 0.08407289534807205,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": 16.34904670715332,
      "logits/rejected": 16.30767059326172,
      "logps/chosen": -0.3302846848964691,
      "logps/rejected": -0.2799247205257416,
      "loss": 1.0285,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": -0.4954269826412201,
      "rewards/margins": -0.07553993165493011,
      "rewards/rejected": -0.41988706588745117,
      "step": 100
    },
    {
      "epoch": 0.15695507161075142,
      "eval_logits/chosen": 16.43848419189453,
      "eval_logits/rejected": 15.978095054626465,
      "eval_logps/chosen": -0.3256901502609253,
      "eval_logps/rejected": -0.2794351279735565,
      "eval_loss": 1.0174708366394043,
      "eval_rewards/accuracies": 0.2884615361690521,
      "eval_rewards/chosen": -0.4885352551937103,
      "eval_rewards/margins": -0.06938254088163376,
      "eval_rewards/rejected": -0.41915270686149597,
      "eval_runtime": 14.607,
      "eval_samples_per_second": 28.206,
      "eval_steps_per_second": 3.56,
      "step": 100
    },
    {
      "epoch": 0.17265057877182657,
      "grad_norm": 0.07448805868625641,
      "learning_rate": 4.933947257182901e-06,
      "logits/chosen": 16.681476593017578,
      "logits/rejected": 16.208908081054688,
      "logps/chosen": -0.3410753309726715,
      "logps/rejected": -0.28177398443222046,
      "loss": 1.0364,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": -0.5116130113601685,
      "rewards/margins": -0.08895199000835419,
      "rewards/rejected": -0.4226610064506531,
      "step": 110
    },
    {
      "epoch": 0.1883460859329017,
      "grad_norm": 0.17266640067100525,
      "learning_rate": 4.921457902821578e-06,
      "logits/chosen": 16.73276710510254,
      "logits/rejected": 16.46255111694336,
      "logps/chosen": -0.30808666348457336,
      "logps/rejected": -0.2663067877292633,
      "loss": 1.0242,
      "rewards/accuracies": 0.23749999701976776,
      "rewards/chosen": -0.46213001012802124,
      "rewards/margins": -0.0626697838306427,
      "rewards/rejected": -0.39946022629737854,
      "step": 120
    },
    {
      "epoch": 0.20404159309397685,
      "grad_norm": 0.1035998985171318,
      "learning_rate": 4.907906416994146e-06,
      "logits/chosen": 16.380239486694336,
      "logits/rejected": 16.233612060546875,
      "logps/chosen": -0.32250356674194336,
      "logps/rejected": -0.27460020780563354,
      "loss": 1.0162,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": -0.48375529050827026,
      "rewards/margins": -0.0718550831079483,
      "rewards/rejected": -0.41190028190612793,
      "step": 130
    },
    {
      "epoch": 0.219737100255052,
      "grad_norm": 0.0961030125617981,
      "learning_rate": 4.893298743830168e-06,
      "logits/chosen": 16.440303802490234,
      "logits/rejected": 16.233903884887695,
      "logps/chosen": -0.33421438932418823,
      "logps/rejected": -0.2857271134853363,
      "loss": 1.0008,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.5013214945793152,
      "rewards/margins": -0.07273083180189133,
      "rewards/rejected": -0.42859068512916565,
      "step": 140
    },
    {
      "epoch": 0.23543260741612712,
      "grad_norm": 0.14799565076828003,
      "learning_rate": 4.8776412907378845e-06,
      "logits/chosen": 16.600738525390625,
      "logits/rejected": 16.536447525024414,
      "logps/chosen": -0.3243326246738434,
      "logps/rejected": -0.3061785101890564,
      "loss": 1.0085,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -0.4864989221096039,
      "rewards/margins": -0.027231160551309586,
      "rewards/rejected": -0.459267795085907,
      "step": 150
    },
    {
      "epoch": 0.23543260741612712,
      "eval_logits/chosen": 16.8078556060791,
      "eval_logits/rejected": 16.311922073364258,
      "eval_logps/chosen": -0.326242595911026,
      "eval_logps/rejected": -0.3398977816104889,
      "eval_loss": 0.9791463017463684,
      "eval_rewards/accuracies": 0.4038461446762085,
      "eval_rewards/chosen": -0.4893638789653778,
      "eval_rewards/margins": 0.020482787862420082,
      "eval_rewards/rejected": -0.5098467469215393,
      "eval_runtime": 14.6072,
      "eval_samples_per_second": 28.205,
      "eval_steps_per_second": 3.56,
      "step": 150
    },
    {
      "epoch": 0.2511281145772023,
      "grad_norm": 0.11464422941207886,
      "learning_rate": 4.860940925593703e-06,
      "logits/chosen": 16.586620330810547,
      "logits/rejected": 16.253028869628906,
      "logps/chosen": -0.33331993222236633,
      "logps/rejected": -0.35580095648765564,
      "loss": 0.9827,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.49997982382774353,
      "rewards/margins": 0.03372158855199814,
      "rewards/rejected": -0.5337014198303223,
      "step": 160
    },
    {
      "epoch": 0.2668236217382774,
      "grad_norm": 0.09462621062994003,
      "learning_rate": 4.84320497372973e-06,
      "logits/chosen": 16.220312118530273,
      "logits/rejected": 16.105356216430664,
      "logps/chosen": -0.2957116663455963,
      "logps/rejected": -0.3528788089752197,
      "loss": 0.9716,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.44356757402420044,
      "rewards/margins": 0.08575066924095154,
      "rewards/rejected": -0.5293182134628296,
      "step": 170
    },
    {
      "epoch": 0.28251912889935255,
      "grad_norm": 0.10995513945817947,
      "learning_rate": 4.824441214720629e-06,
      "logits/chosen": 16.811044692993164,
      "logits/rejected": 16.43205451965332,
      "logps/chosen": -0.3263992369174957,
      "logps/rejected": -0.35878634452819824,
      "loss": 0.9546,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.4895988404750824,
      "rewards/margins": 0.04858064278960228,
      "rewards/rejected": -0.5381795167922974,
      "step": 180
    },
    {
      "epoch": 0.2982146360604277,
      "grad_norm": 0.1348036825656891,
      "learning_rate": 4.804657878971252e-06,
      "logits/chosen": 16.816158294677734,
      "logits/rejected": 16.55413055419922,
      "logps/chosen": -0.31842875480651855,
      "logps/rejected": -0.3791810870170593,
      "loss": 0.9472,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.47764310240745544,
      "rewards/margins": 0.09112847596406937,
      "rewards/rejected": -0.5687715411186218,
      "step": 190
    },
    {
      "epoch": 0.31391014322150285,
      "grad_norm": 0.13393694162368774,
      "learning_rate": 4.783863644106502e-06,
      "logits/chosen": 17.075885772705078,
      "logits/rejected": 16.982004165649414,
      "logps/chosen": -0.3368823528289795,
      "logps/rejected": -0.3636801540851593,
      "loss": 0.9455,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5053235292434692,
      "rewards/margins": 0.04019671678543091,
      "rewards/rejected": -0.5455202460289001,
      "step": 200
    },
    {
      "epoch": 0.31391014322150285,
      "eval_logits/chosen": 17.17239761352539,
      "eval_logits/rejected": 16.609193801879883,
      "eval_logps/chosen": -0.33508703112602234,
      "eval_logps/rejected": -0.4493505656719208,
      "eval_loss": 0.9283667802810669,
      "eval_rewards/accuracies": 0.4615384638309479,
      "eval_rewards/chosen": -0.5026305913925171,
      "eval_rewards/margins": 0.17139528691768646,
      "eval_rewards/rejected": -0.67402583360672,
      "eval_runtime": 14.5913,
      "eval_samples_per_second": 28.236,
      "eval_steps_per_second": 3.564,
      "step": 200
    },
    {
      "epoch": 0.329605650382578,
      "grad_norm": 0.14985737204551697,
      "learning_rate": 4.762067631165049e-06,
      "logits/chosen": 16.92913246154785,
      "logits/rejected": 16.648632049560547,
      "logps/chosen": -0.3326043486595154,
      "logps/rejected": -0.4218205511569977,
      "loss": 0.9225,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.4989064335823059,
      "rewards/margins": 0.13382436335086823,
      "rewards/rejected": -0.6327308416366577,
      "step": 210
    },
    {
      "epoch": 0.34530115754365315,
      "grad_norm": 0.16972233355045319,
      "learning_rate": 4.7392794005985324e-06,
      "logits/chosen": 17.074562072753906,
      "logits/rejected": 16.863161087036133,
      "logps/chosen": -0.3645358383655548,
      "logps/rejected": -0.5221719145774841,
      "loss": 0.9264,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.5468038320541382,
      "rewards/margins": 0.23645417392253876,
      "rewards/rejected": -0.7832580208778381,
      "step": 220
    },
    {
      "epoch": 0.3609966647047283,
      "grad_norm": 0.20796354115009308,
      "learning_rate": 4.715508948078037e-06,
      "logits/chosen": 17.4168643951416,
      "logits/rejected": 17.00813102722168,
      "logps/chosen": -0.35443297028541565,
      "logps/rejected": -0.5115998983383179,
      "loss": 0.9037,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.5316494703292847,
      "rewards/margins": 0.23575039207935333,
      "rewards/rejected": -0.767399787902832,
      "step": 230
    },
    {
      "epoch": 0.3766921718658034,
      "grad_norm": 0.1932348757982254,
      "learning_rate": 4.690766700109659e-06,
      "logits/chosen": 17.034719467163086,
      "logits/rejected": 16.65166664123535,
      "logps/chosen": -0.3426091969013214,
      "logps/rejected": -0.47884297370910645,
      "loss": 0.9154,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5139138102531433,
      "rewards/margins": 0.20435063540935516,
      "rewards/rejected": -0.7182644605636597,
      "step": 240
    },
    {
      "epoch": 0.39238767902687854,
      "grad_norm": 1.016450047492981,
      "learning_rate": 4.665063509461098e-06,
      "logits/chosen": 16.935081481933594,
      "logits/rejected": 16.65024185180664,
      "logps/chosen": -0.41738080978393555,
      "logps/rejected": -0.48515433073043823,
      "loss": 0.8774,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.6260712146759033,
      "rewards/margins": 0.10166029632091522,
      "rewards/rejected": -0.7277315258979797,
      "step": 250
    },
    {
      "epoch": 0.39238767902687854,
      "eval_logits/chosen": 17.221881866455078,
      "eval_logits/rejected": 16.536666870117188,
      "eval_logps/chosen": -0.3716265559196472,
      "eval_logps/rejected": -0.87992262840271,
      "eval_loss": 0.8047741651535034,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/chosen": -0.5574398636817932,
      "eval_rewards/margins": 0.7624441981315613,
      "eval_rewards/rejected": -1.3198840618133545,
      "eval_runtime": 14.6008,
      "eval_samples_per_second": 28.218,
      "eval_steps_per_second": 3.561,
      "step": 250
    },
    {
      "epoch": 0.4080831861879537,
      "grad_norm": 0.2831684648990631,
      "learning_rate": 4.638410650401267e-06,
      "logits/chosen": 17.084096908569336,
      "logits/rejected": 16.564823150634766,
      "logps/chosen": -0.38100525736808777,
      "logps/rejected": -0.8173269033432007,
      "loss": 0.8197,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5715079307556152,
      "rewards/margins": 0.6544824838638306,
      "rewards/rejected": -1.2259904146194458,
      "step": 260
    },
    {
      "epoch": 0.42377869334902885,
      "grad_norm": 0.9195305109024048,
      "learning_rate": 4.610819813755038e-06,
      "logits/chosen": 17.316537857055664,
      "logits/rejected": 16.623016357421875,
      "logps/chosen": -0.4713365435600281,
      "logps/rejected": -1.2184536457061768,
      "loss": 0.7493,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.7070047855377197,
      "rewards/margins": 1.120675802230835,
      "rewards/rejected": -1.8276805877685547,
      "step": 270
    },
    {
      "epoch": 0.439474200510104,
      "grad_norm": 0.5542411804199219,
      "learning_rate": 4.582303101775249e-06,
      "logits/chosen": 17.173168182373047,
      "logits/rejected": 16.759702682495117,
      "logps/chosen": -0.4629506468772888,
      "logps/rejected": -1.3020483255386353,
      "loss": 0.7638,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6944260597229004,
      "rewards/margins": 1.2586463689804077,
      "rewards/rejected": -1.9530725479125977,
      "step": 280
    },
    {
      "epoch": 0.45516970767117915,
      "grad_norm": 0.2836654782295227,
      "learning_rate": 4.55287302283426e-06,
      "logits/chosen": 17.022424697875977,
      "logits/rejected": 16.43834114074707,
      "logps/chosen": -0.527222216129303,
      "logps/rejected": -1.632965087890625,
      "loss": 0.7466,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7908332943916321,
      "rewards/margins": 1.6586145162582397,
      "rewards/rejected": -2.4494476318359375,
      "step": 290
    },
    {
      "epoch": 0.47086521483225424,
      "grad_norm": 0.658358097076416,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": 16.6541690826416,
      "logits/rejected": 16.34103012084961,
      "logps/chosen": -0.50589519739151,
      "logps/rejected": -1.8910831212997437,
      "loss": 0.6982,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7588427662849426,
      "rewards/margins": 2.077782154083252,
      "rewards/rejected": -2.8366246223449707,
      "step": 300
    },
    {
      "epoch": 0.47086521483225424,
      "eval_logits/chosen": 16.950870513916016,
      "eval_logits/rejected": 16.154659271240234,
      "eval_logps/chosen": -0.5594518184661865,
      "eval_logps/rejected": -1.9617934226989746,
      "eval_loss": 0.7422243356704712,
      "eval_rewards/accuracies": 0.7115384340286255,
      "eval_rewards/chosen": -0.839177668094635,
      "eval_rewards/margins": 2.103512763977051,
      "eval_rewards/rejected": -2.942690372467041,
      "eval_runtime": 14.5969,
      "eval_samples_per_second": 28.225,
      "eval_steps_per_second": 3.562,
      "step": 300
    },
    {
      "epoch": 0.4865607219933294,
      "grad_norm": 1.1044840812683105,
      "learning_rate": 4.491324795060491e-06,
      "logits/chosen": 17.233802795410156,
      "logits/rejected": 16.292484283447266,
      "logps/chosen": -0.5940151214599609,
      "logps/rejected": -2.041588306427002,
      "loss": 0.7415,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.8910226821899414,
      "rewards/margins": 2.1713600158691406,
      "rewards/rejected": -3.062382459640503,
      "step": 310
    },
    {
      "epoch": 0.5022562291544046,
      "grad_norm": 0.524356484413147,
      "learning_rate": 4.4592336433146e-06,
      "logits/chosen": 17.155719757080078,
      "logits/rejected": 16.328996658325195,
      "logps/chosen": -0.6095865964889526,
      "logps/rejected": -1.831730604171753,
      "loss": 0.7249,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9143797755241394,
      "rewards/margins": 1.8332160711288452,
      "rewards/rejected": -2.747596025466919,
      "step": 320
    },
    {
      "epoch": 0.5179517363154797,
      "grad_norm": 0.4383145570755005,
      "learning_rate": 4.426283106939474e-06,
      "logits/chosen": 16.774551391601562,
      "logits/rejected": 16.532238006591797,
      "logps/chosen": -0.669482946395874,
      "logps/rejected": -1.7058660984039307,
      "loss": 0.6868,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.004224419593811,
      "rewards/margins": 1.5545748472213745,
      "rewards/rejected": -2.5587992668151855,
      "step": 330
    },
    {
      "epoch": 0.5336472434765548,
      "grad_norm": 3.194026231765747,
      "learning_rate": 4.3924876391293915e-06,
      "logits/chosen": 16.963787078857422,
      "logits/rejected": 16.3092041015625,
      "logps/chosen": -0.7091449499130249,
      "logps/rejected": -2.1812453269958496,
      "loss": 0.706,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0637174844741821,
      "rewards/margins": 2.208150863647461,
      "rewards/rejected": -3.2718684673309326,
      "step": 340
    },
    {
      "epoch": 0.5493427506376299,
      "grad_norm": 0.5639687776565552,
      "learning_rate": 4.357862063693486e-06,
      "logits/chosen": 17.019062042236328,
      "logits/rejected": 16.315940856933594,
      "logps/chosen": -0.8996315002441406,
      "logps/rejected": -2.0307648181915283,
      "loss": 0.674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.349447250366211,
      "rewards/margins": 1.696699857711792,
      "rewards/rejected": -3.046147108078003,
      "step": 350
    },
    {
      "epoch": 0.5493427506376299,
      "eval_logits/chosen": 16.70261001586914,
      "eval_logits/rejected": 15.845681190490723,
      "eval_logps/chosen": -0.9138904809951782,
      "eval_logps/rejected": -2.5003392696380615,
      "eval_loss": 0.6938430070877075,
      "eval_rewards/accuracies": 0.8461538553237915,
      "eval_rewards/chosen": -1.3708356618881226,
      "eval_rewards/margins": 2.3796732425689697,
      "eval_rewards/rejected": -3.75050950050354,
      "eval_runtime": 14.6072,
      "eval_samples_per_second": 28.205,
      "eval_steps_per_second": 3.56,
      "step": 350
    },
    {
      "epoch": 0.5650382577987051,
      "grad_norm": 0.6917555928230286,
      "learning_rate": 4.322421568553529e-06,
      "logits/chosen": 16.269399642944336,
      "logits/rejected": 15.73193645477295,
      "logps/chosen": -1.0171478986740112,
      "logps/rejected": -2.201298236846924,
      "loss": 0.651,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.525721788406372,
      "rewards/margins": 1.7762253284454346,
      "rewards/rejected": -3.3019473552703857,
      "step": 360
    },
    {
      "epoch": 0.5807337649597802,
      "grad_norm": 1.151584267616272,
      "learning_rate": 4.286181699082008e-06,
      "logits/chosen": 15.88188648223877,
      "logits/rejected": 15.261810302734375,
      "logps/chosen": -1.2272206544876099,
      "logps/rejected": -3.0203287601470947,
      "loss": 0.665,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8408310413360596,
      "rewards/margins": 2.689661741256714,
      "rewards/rejected": -4.530492305755615,
      "step": 370
    },
    {
      "epoch": 0.5964292721208554,
      "grad_norm": 1.1879621744155884,
      "learning_rate": 4.249158351283414e-06,
      "logits/chosen": 15.906834602355957,
      "logits/rejected": 15.42895221710205,
      "logps/chosen": -1.5027008056640625,
      "logps/rejected": -2.9172558784484863,
      "loss": 0.5998,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.2540509700775146,
      "rewards/margins": 2.1218326091766357,
      "rewards/rejected": -4.37588357925415,
      "step": 380
    },
    {
      "epoch": 0.6121247792819305,
      "grad_norm": 1.678884744644165,
      "learning_rate": 4.211367764821722e-06,
      "logits/chosen": 15.630645751953125,
      "logits/rejected": 14.888415336608887,
      "logps/chosen": -2.231367349624634,
      "logps/rejected": -3.5754780769348145,
      "loss": 0.5937,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.347050905227661,
      "rewards/margins": 2.0161664485931396,
      "rewards/rejected": -5.363216876983643,
      "step": 390
    },
    {
      "epoch": 0.6278202864430057,
      "grad_norm": 2.008671998977661,
      "learning_rate": 4.172826515897146e-06,
      "logits/chosen": 15.024614334106445,
      "logits/rejected": 14.981277465820312,
      "logps/chosen": -2.665980815887451,
      "logps/rejected": -3.582763195037842,
      "loss": 0.5722,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.998971462249756,
      "rewards/margins": 1.3751739263534546,
      "rewards/rejected": -5.374145030975342,
      "step": 400
    },
    {
      "epoch": 0.6278202864430057,
      "eval_logits/chosen": 15.45953369140625,
      "eval_logits/rejected": 14.663763046264648,
      "eval_logps/chosen": -2.7269070148468018,
      "eval_logps/rejected": -4.235719680786133,
      "eval_loss": 0.5668805837631226,
      "eval_rewards/accuracies": 0.8846153616905212,
      "eval_rewards/chosen": -4.090360641479492,
      "eval_rewards/margins": 2.263219118118286,
      "eval_rewards/rejected": -6.353579998016357,
      "eval_runtime": 14.6036,
      "eval_samples_per_second": 28.212,
      "eval_steps_per_second": 3.561,
      "step": 400
    },
    {
      "epoch": 0.6435157936040808,
      "grad_norm": 1.018362045288086,
      "learning_rate": 4.133551509975264e-06,
      "logits/chosen": 15.027656555175781,
      "logits/rejected": 14.74786376953125,
      "logps/chosen": -2.76598858833313,
      "logps/rejected": -4.335003852844238,
      "loss": 0.5412,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -4.148982524871826,
      "rewards/margins": 2.3535237312316895,
      "rewards/rejected": -6.502506256103516,
      "step": 410
    },
    {
      "epoch": 0.659211300765156,
      "grad_norm": 2.2899765968322754,
      "learning_rate": 4.093559974371725e-06,
      "logits/chosen": 15.15197467803955,
      "logits/rejected": 14.681947708129883,
      "logps/chosen": -2.9831995964050293,
      "logps/rejected": -4.723050117492676,
      "loss": 0.4652,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -4.474799633026123,
      "rewards/margins": 2.6097757816314697,
      "rewards/rejected": -7.0845746994018555,
      "step": 420
    },
    {
      "epoch": 0.6749068079262311,
      "grad_norm": 1.419196605682373,
      "learning_rate": 4.052869450695776e-06,
      "logits/chosen": 15.000317573547363,
      "logits/rejected": 14.538591384887695,
      "logps/chosen": -3.543004274368286,
      "logps/rejected": -5.037484169006348,
      "loss": 0.427,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.314507007598877,
      "rewards/margins": 2.2417192459106445,
      "rewards/rejected": -7.556225776672363,
      "step": 430
    },
    {
      "epoch": 0.6906023150873063,
      "grad_norm": 1.639215111732483,
      "learning_rate": 4.011497787155938e-06,
      "logits/chosen": 14.798017501831055,
      "logits/rejected": 14.236108779907227,
      "logps/chosen": -4.154418468475342,
      "logps/rejected": -6.0606184005737305,
      "loss": 0.4692,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -6.231626987457275,
      "rewards/margins": 2.859299421310425,
      "rewards/rejected": -9.090926170349121,
      "step": 440
    },
    {
      "epoch": 0.7062978222483814,
      "grad_norm": 1.6229957342147827,
      "learning_rate": 3.969463130731183e-06,
      "logits/chosen": 14.495908737182617,
      "logits/rejected": 13.978658676147461,
      "logps/chosen": -3.4806721210479736,
      "logps/rejected": -5.746390342712402,
      "loss": 0.4218,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.221007823944092,
      "rewards/margins": 3.3985772132873535,
      "rewards/rejected": -8.619585037231445,
      "step": 450
    },
    {
      "epoch": 0.7062978222483814,
      "eval_logits/chosen": 15.033548355102539,
      "eval_logits/rejected": 14.20669174194336,
      "eval_logps/chosen": -3.423999547958374,
      "eval_logps/rejected": -5.855647563934326,
      "eval_loss": 0.45794492959976196,
      "eval_rewards/accuracies": 0.942307710647583,
      "eval_rewards/chosen": -5.135998725891113,
      "eval_rewards/margins": 3.6474733352661133,
      "eval_rewards/rejected": -8.783472061157227,
      "eval_runtime": 14.6088,
      "eval_samples_per_second": 28.202,
      "eval_steps_per_second": 3.559,
      "step": 450
    },
    {
      "epoch": 0.7219933294094566,
      "grad_norm": 1.7993106842041016,
      "learning_rate": 3.92678391921108e-06,
      "logits/chosen": 14.569323539733887,
      "logits/rejected": 13.937005996704102,
      "logps/chosen": -3.3623528480529785,
      "logps/rejected": -5.784353733062744,
      "loss": 0.3675,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.043529033660889,
      "rewards/margins": 3.6330013275146484,
      "rewards/rejected": -8.676530838012695,
      "step": 460
    },
    {
      "epoch": 0.7376888365705316,
      "grad_norm": 2.1828088760375977,
      "learning_rate": 3.88347887310836e-06,
      "logits/chosen": 13.90056037902832,
      "logits/rejected": 13.489587783813477,
      "logps/chosen": -3.959341049194336,
      "logps/rejected": -6.217314720153809,
      "loss": 0.3653,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -5.939011573791504,
      "rewards/margins": 3.3869614601135254,
      "rewards/rejected": -9.325971603393555,
      "step": 470
    },
    {
      "epoch": 0.7533843437316068,
      "grad_norm": 2.128547191619873,
      "learning_rate": 3.839566987447492e-06,
      "logits/chosen": 14.113011360168457,
      "logits/rejected": 13.571691513061523,
      "logps/chosen": -4.004420280456543,
      "logps/rejected": -6.216452121734619,
      "loss": 0.3439,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.0066304206848145,
      "rewards/margins": 3.3180477619171143,
      "rewards/rejected": -9.324677467346191,
      "step": 480
    },
    {
      "epoch": 0.7690798508926819,
      "grad_norm": 3.2651569843292236,
      "learning_rate": 3.795067523432826e-06,
      "logits/chosen": 13.918218612670898,
      "logits/rejected": 13.147412300109863,
      "logps/chosen": -4.574521064758301,
      "logps/rejected": -7.411087989807129,
      "loss": 0.3645,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.861782073974609,
      "rewards/margins": 4.254849910736084,
      "rewards/rejected": -11.116632461547852,
      "step": 490
    },
    {
      "epoch": 0.7847753580537571,
      "grad_norm": 2.163595676422119,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": 13.972379684448242,
      "logits/rejected": 13.390825271606445,
      "logps/chosen": -3.931866407394409,
      "logps/rejected": -6.547454833984375,
      "loss": 0.3698,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.897799491882324,
      "rewards/margins": 3.923382520675659,
      "rewards/rejected": -9.821184158325195,
      "step": 500
    },
    {
      "epoch": 0.7847753580537571,
      "eval_logits/chosen": 14.325434684753418,
      "eval_logits/rejected": 13.4517822265625,
      "eval_logps/chosen": -3.7717440128326416,
      "eval_logps/rejected": -6.710938453674316,
      "eval_loss": 0.38915327191352844,
      "eval_rewards/accuracies": 0.942307710647583,
      "eval_rewards/chosen": -5.657615661621094,
      "eval_rewards/margins": 4.408792018890381,
      "eval_rewards/rejected": -10.066408157348633,
      "eval_runtime": 14.5996,
      "eval_samples_per_second": 28.22,
      "eval_steps_per_second": 3.562,
      "step": 500
    },
    {
      "epoch": 0.8004708652148322,
      "grad_norm": 2.7098453044891357,
      "learning_rate": 3.7043841852542884e-06,
      "logits/chosen": 13.909749984741211,
      "logits/rejected": 13.060315132141113,
      "logps/chosen": -3.848449230194092,
      "logps/rejected": -6.747067928314209,
      "loss": 0.3547,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.772673606872559,
      "rewards/margins": 4.347928524017334,
      "rewards/rejected": -10.120603561401367,
      "step": 510
    },
    {
      "epoch": 0.8161663723759074,
      "grad_norm": 1.693467140197754,
      "learning_rate": 3.658240087799655e-06,
      "logits/chosen": 13.52336311340332,
      "logits/rejected": 13.026924133300781,
      "logps/chosen": -4.218991279602051,
      "logps/rejected": -6.984036445617676,
      "loss": 0.3718,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.328486442565918,
      "rewards/margins": 4.1475677490234375,
      "rewards/rejected": -10.476054191589355,
      "step": 520
    },
    {
      "epoch": 0.8318618795369825,
      "grad_norm": 2.4496915340423584,
      "learning_rate": 3.611587947962319e-06,
      "logits/chosen": 13.840197563171387,
      "logits/rejected": 13.181543350219727,
      "logps/chosen": -4.708582401275635,
      "logps/rejected": -7.19864559173584,
      "loss": 0.329,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.062872886657715,
      "rewards/margins": 3.7350947856903076,
      "rewards/rejected": -10.797967910766602,
      "step": 530
    },
    {
      "epoch": 0.8475573866980577,
      "grad_norm": 5.772508144378662,
      "learning_rate": 3.564448228912682e-06,
      "logits/chosen": 13.432130813598633,
      "logits/rejected": 12.825490951538086,
      "logps/chosen": -4.506819248199463,
      "logps/rejected": -7.506214141845703,
      "loss": 0.2544,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.760228633880615,
      "rewards/margins": 4.499091625213623,
      "rewards/rejected": -11.259321212768555,
      "step": 540
    },
    {
      "epoch": 0.8632528938591328,
      "grad_norm": 1.7776503562927246,
      "learning_rate": 3.516841607689501e-06,
      "logits/chosen": 13.536295890808105,
      "logits/rejected": 12.703726768493652,
      "logps/chosen": -4.724976062774658,
      "logps/rejected": -7.864454746246338,
      "loss": 0.2795,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.087464332580566,
      "rewards/margins": 4.7092180252075195,
      "rewards/rejected": -11.796682357788086,
      "step": 550
    },
    {
      "epoch": 0.8632528938591328,
      "eval_logits/chosen": 13.692171096801758,
      "eval_logits/rejected": 12.81371021270752,
      "eval_logps/chosen": -4.32358980178833,
      "eval_logps/rejected": -7.658167839050293,
      "eval_loss": 0.335475891828537,
      "eval_rewards/accuracies": 0.942307710647583,
      "eval_rewards/chosen": -6.485384464263916,
      "eval_rewards/margins": 5.00186824798584,
      "eval_rewards/rejected": -11.487252235412598,
      "eval_runtime": 14.6093,
      "eval_samples_per_second": 28.201,
      "eval_steps_per_second": 3.559,
      "step": 550
    },
    {
      "epoch": 0.878948401020208,
      "grad_norm": 1.7613197565078735,
      "learning_rate": 3.4687889661302577e-06,
      "logits/chosen": 13.689091682434082,
      "logits/rejected": 12.911500930786133,
      "logps/chosen": -3.8375942707061768,
      "logps/rejected": -6.98319149017334,
      "loss": 0.3008,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.7563910484313965,
      "rewards/margins": 4.7183966636657715,
      "rewards/rejected": -10.474787712097168,
      "step": 560
    },
    {
      "epoch": 0.8946439081812831,
      "grad_norm": 2.1285247802734375,
      "learning_rate": 3.4203113817116955e-06,
      "logits/chosen": 13.902356147766113,
      "logits/rejected": 12.766775131225586,
      "logps/chosen": -3.7727108001708984,
      "logps/rejected": -7.6362409591674805,
      "loss": 0.2816,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.6590657234191895,
      "rewards/margins": 5.795297622680664,
      "rewards/rejected": -11.454363822937012,
      "step": 570
    },
    {
      "epoch": 0.9103394153423583,
      "grad_norm": 1.6965669393539429,
      "learning_rate": 3.3714301183045382e-06,
      "logits/chosen": 13.374112129211426,
      "logits/rejected": 12.414613723754883,
      "logps/chosen": -4.747832298278809,
      "logps/rejected": -8.099973678588867,
      "loss": 0.2921,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -7.121748924255371,
      "rewards/margins": 5.028212547302246,
      "rewards/rejected": -12.1499605178833,
      "step": 580
    },
    {
      "epoch": 0.9260349225034334,
      "grad_norm": 2.5470242500305176,
      "learning_rate": 3.3221666168464584e-06,
      "logits/chosen": 13.287660598754883,
      "logits/rejected": 12.554139137268066,
      "logps/chosen": -4.818625450134277,
      "logps/rejected": -8.005637168884277,
      "loss": 0.3155,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.227938175201416,
      "rewards/margins": 4.780518531799316,
      "rewards/rejected": -12.008456230163574,
      "step": 590
    },
    {
      "epoch": 0.9417304296645085,
      "grad_norm": 2.374859571456909,
      "learning_rate": 3.272542485937369e-06,
      "logits/chosen": 13.152572631835938,
      "logits/rejected": 12.557012557983398,
      "logps/chosen": -3.7581982612609863,
      "logps/rejected": -6.985205173492432,
      "loss": 0.2565,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.6372971534729,
      "rewards/margins": 4.840510368347168,
      "rewards/rejected": -10.477807998657227,
      "step": 600
    },
    {
      "epoch": 0.9417304296645085,
      "eval_logits/chosen": 13.585193634033203,
      "eval_logits/rejected": 12.657953262329102,
      "eval_logps/chosen": -4.077207088470459,
      "eval_logps/rejected": -7.660572528839111,
      "eval_loss": 0.30504748225212097,
      "eval_rewards/accuracies": 0.9615384340286255,
      "eval_rewards/chosen": -6.115810871124268,
      "eval_rewards/margins": 5.375046730041504,
      "eval_rewards/rejected": -11.49085807800293,
      "eval_runtime": 14.6046,
      "eval_samples_per_second": 28.21,
      "eval_steps_per_second": 3.561,
      "step": 600
    },
    {
      "epoch": 0.9574259368255836,
      "grad_norm": 3.3574252128601074,
      "learning_rate": 3.222579492361179e-06,
      "logits/chosen": 12.703638076782227,
      "logits/rejected": 11.910614013671875,
      "logps/chosen": -4.188838005065918,
      "logps/rejected": -7.538239479064941,
      "loss": 0.2527,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.283257961273193,
      "rewards/margins": 5.024100303649902,
      "rewards/rejected": -11.30735969543457,
      "step": 610
    },
    {
      "epoch": 0.9731214439866588,
      "grad_norm": 1.726338505744934,
      "learning_rate": 3.1722995515381644e-06,
      "logits/chosen": 13.002418518066406,
      "logits/rejected": 12.060869216918945,
      "logps/chosen": -4.711850166320801,
      "logps/rejected": -8.62987995147705,
      "loss": 0.2895,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.067776679992676,
      "rewards/margins": 5.877043724060059,
      "rewards/rejected": -12.944819450378418,
      "step": 620
    },
    {
      "epoch": 0.9888169511477339,
      "grad_norm": 2.5146796703338623,
      "learning_rate": 3.121724717912138e-06,
      "logits/chosen": 13.190832138061523,
      "logits/rejected": 11.927602767944336,
      "logps/chosen": -4.228806495666504,
      "logps/rejected": -8.436616897583008,
      "loss": 0.2284,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.343210220336914,
      "rewards/margins": 6.311715602874756,
      "rewards/rejected": -12.654926300048828,
      "step": 630
    },
    {
      "epoch": 1.003139101432215,
      "grad_norm": 4.229348659515381,
      "learning_rate": 3.0708771752766397e-06,
      "logits/chosen": 13.159438133239746,
      "logits/rejected": 12.417133331298828,
      "logps/chosen": -4.084325313568115,
      "logps/rejected": -7.705818176269531,
      "loss": 0.2276,
      "rewards/accuracies": 0.9178082346916199,
      "rewards/chosen": -6.126489162445068,
      "rewards/margins": 5.432239055633545,
      "rewards/rejected": -11.558727264404297,
      "step": 640
    },
    {
      "epoch": 1.0188346085932902,
      "grad_norm": 4.93217134475708,
      "learning_rate": 3.019779227044398e-06,
      "logits/chosen": 13.347890853881836,
      "logits/rejected": 12.535311698913574,
      "logps/chosen": -4.311320781707764,
      "logps/rejected": -7.402029514312744,
      "loss": 0.2353,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.466981410980225,
      "rewards/margins": 4.636063098907471,
      "rewards/rejected": -11.103044509887695,
      "step": 650
    },
    {
      "epoch": 1.0188346085932902,
      "eval_logits/chosen": 13.296299934387207,
      "eval_logits/rejected": 12.358522415161133,
      "eval_logps/chosen": -4.177649974822998,
      "eval_logps/rejected": -8.009714126586914,
      "eval_loss": 0.28009530901908875,
      "eval_rewards/accuracies": 0.9615384340286255,
      "eval_rewards/chosen": -6.266475200653076,
      "eval_rewards/margins": 5.748095989227295,
      "eval_rewards/rejected": -12.014572143554688,
      "eval_runtime": 14.6091,
      "eval_samples_per_second": 28.202,
      "eval_steps_per_second": 3.559,
      "step": 650
    },
    {
      "epoch": 1.0345301157543654,
      "grad_norm": 2.0475847721099854,
      "learning_rate": 2.9684532864643123e-06,
      "logits/chosen": 13.43058967590332,
      "logits/rejected": 12.661652565002441,
      "logps/chosen": -3.9617011547088623,
      "logps/rejected": -7.092957973480225,
      "loss": 0.2562,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.942551136016846,
      "rewards/margins": 4.69688606262207,
      "rewards/rejected": -10.639436721801758,
      "step": 660
    },
    {
      "epoch": 1.0502256229154405,
      "grad_norm": 4.578652381896973,
      "learning_rate": 2.9169218667902562e-06,
      "logits/chosen": 13.032757759094238,
      "logits/rejected": 12.292248725891113,
      "logps/chosen": -3.679547071456909,
      "logps/rejected": -7.649219512939453,
      "loss": 0.2271,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.519320487976074,
      "rewards/margins": 5.9545087814331055,
      "rewards/rejected": -11.47382926940918,
      "step": 670
    },
    {
      "epoch": 1.0659211300765157,
      "grad_norm": 3.0983850955963135,
      "learning_rate": 2.8652075714060296e-06,
      "logits/chosen": 12.6952486038208,
      "logits/rejected": 11.917645454406738,
      "logps/chosen": -4.468019008636475,
      "logps/rejected": -8.505170822143555,
      "loss": 0.179,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.702028751373291,
      "rewards/margins": 6.055727958679199,
      "rewards/rejected": -12.757757186889648,
      "step": 680
    },
    {
      "epoch": 1.0816166372375908,
      "grad_norm": 3.071026086807251,
      "learning_rate": 2.813333083910761e-06,
      "logits/chosen": 12.464409828186035,
      "logits/rejected": 11.680208206176758,
      "logps/chosen": -4.6339874267578125,
      "logps/rejected": -8.639189720153809,
      "loss": 0.2541,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.950981140136719,
      "rewards/margins": 6.007803440093994,
      "rewards/rejected": -12.958784103393555,
      "step": 690
    },
    {
      "epoch": 1.097312144398666,
      "grad_norm": 1.3732093572616577,
      "learning_rate": 2.761321158169134e-06,
      "logits/chosen": 12.308462142944336,
      "logits/rejected": 11.674012184143066,
      "logps/chosen": -4.638761043548584,
      "logps/rejected": -8.936636924743652,
      "loss": 0.1919,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.958141326904297,
      "rewards/margins": 6.446814060211182,
      "rewards/rejected": -13.40495491027832,
      "step": 700
    },
    {
      "epoch": 1.097312144398666,
      "eval_logits/chosen": 13.217435836791992,
      "eval_logits/rejected": 12.272343635559082,
      "eval_logps/chosen": -4.347932815551758,
      "eval_logps/rejected": -8.441193580627441,
      "eval_loss": 0.25916898250579834,
      "eval_rewards/accuracies": 0.9615384340286255,
      "eval_rewards/chosen": -6.521899223327637,
      "eval_rewards/margins": 6.139890670776367,
      "eval_rewards/rejected": -12.661789894104004,
      "eval_runtime": 14.6102,
      "eval_samples_per_second": 28.2,
      "eval_steps_per_second": 3.559,
      "step": 700
    },
    {
      "epoch": 1.113007651559741,
      "grad_norm": 3.5739834308624268,
      "learning_rate": 2.70919460833079e-06,
      "logits/chosen": 12.828997611999512,
      "logits/rejected": 12.09227180480957,
      "logps/chosen": -4.3685622215271,
      "logps/rejected": -8.672820091247559,
      "loss": 0.2187,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.552844047546387,
      "rewards/margins": 6.456387519836426,
      "rewards/rejected": -13.009231567382812,
      "step": 710
    },
    {
      "epoch": 1.1287031587208163,
      "grad_norm": 11.475191116333008,
      "learning_rate": 2.6569762988232838e-06,
      "logits/chosen": 13.344253540039062,
      "logits/rejected": 12.190181732177734,
      "logps/chosen": -4.567076206207275,
      "logps/rejected": -8.719629287719727,
      "loss": 0.1847,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.85061502456665,
      "rewards/margins": 6.228829383850098,
      "rewards/rejected": -13.079442977905273,
      "step": 720
    },
    {
      "epoch": 1.1443986658818912,
      "grad_norm": 1.8145791292190552,
      "learning_rate": 2.604689134322999e-06,
      "logits/chosen": 12.794706344604492,
      "logits/rejected": 12.407793045043945,
      "logps/chosen": -4.447361946105957,
      "logps/rejected": -8.19226360321045,
      "loss": 0.1914,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.67104434967041,
      "rewards/margins": 5.6173505783081055,
      "rewards/rejected": -12.288395881652832,
      "step": 730
    },
    {
      "epoch": 1.1600941730429664,
      "grad_norm": 3.763374090194702,
      "learning_rate": 2.5523560497083927e-06,
      "logits/chosen": 12.97741413116455,
      "logits/rejected": 12.224584579467773,
      "logps/chosen": -4.449431419372559,
      "logps/rejected": -8.755613327026367,
      "loss": 0.2055,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.6741461753845215,
      "rewards/margins": 6.459272861480713,
      "rewards/rejected": -13.133418083190918,
      "step": 740
    },
    {
      "epoch": 1.1757896802040415,
      "grad_norm": 2.816288948059082,
      "learning_rate": 2.5e-06,
      "logits/chosen": 12.736323356628418,
      "logits/rejected": 12.039944648742676,
      "logps/chosen": -4.300374507904053,
      "logps/rejected": -8.570680618286133,
      "loss": 0.185,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.4505615234375,
      "rewards/margins": 6.405457973480225,
      "rewards/rejected": -12.856019973754883,
      "step": 750
    },
    {
      "epoch": 1.1757896802040415,
      "eval_logits/chosen": 13.174198150634766,
      "eval_logits/rejected": 12.189348220825195,
      "eval_logps/chosen": -4.302711009979248,
      "eval_logps/rejected": -8.745194435119629,
      "eval_loss": 0.24307754635810852,
      "eval_rewards/accuracies": 0.9615384340286255,
      "eval_rewards/chosen": -6.454067707061768,
      "eval_rewards/margins": 6.663723945617676,
      "eval_rewards/rejected": -13.117791175842285,
      "eval_runtime": 14.5945,
      "eval_samples_per_second": 28.23,
      "eval_steps_per_second": 3.563,
      "step": 750
    },
    {
      "epoch": 1.1914851873651167,
      "grad_norm": 2.6783711910247803,
      "learning_rate": 2.447643950291608e-06,
      "logits/chosen": 12.992940902709961,
      "logits/rejected": 12.16160774230957,
      "logps/chosen": -4.402331829071045,
      "logps/rejected": -8.848995208740234,
      "loss": 0.1958,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.603497505187988,
      "rewards/margins": 6.6699957847595215,
      "rewards/rejected": -13.273493766784668,
      "step": 760
    },
    {
      "epoch": 1.2071806945261918,
      "grad_norm": 1.580244779586792,
      "learning_rate": 2.3953108656770018e-06,
      "logits/chosen": 12.522805213928223,
      "logits/rejected": 11.664339065551758,
      "logps/chosen": -3.9605941772460938,
      "logps/rejected": -8.103001594543457,
      "loss": 0.1921,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.940891265869141,
      "rewards/margins": 6.213610649108887,
      "rewards/rejected": -12.154502868652344,
      "step": 770
    },
    {
      "epoch": 1.222876201687267,
      "grad_norm": 2.2647345066070557,
      "learning_rate": 2.3430237011767166e-06,
      "logits/chosen": 12.654630661010742,
      "logits/rejected": 11.677509307861328,
      "logps/chosen": -4.634467601776123,
      "logps/rejected": -9.647830963134766,
      "loss": 0.1731,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.951702117919922,
      "rewards/margins": 7.520044803619385,
      "rewards/rejected": -14.471745491027832,
      "step": 780
    },
    {
      "epoch": 1.2385717088483421,
      "grad_norm": 3.8534202575683594,
      "learning_rate": 2.290805391669212e-06,
      "logits/chosen": 12.548233032226562,
      "logits/rejected": 11.633279800415039,
      "logps/chosen": -4.690009593963623,
      "logps/rejected": -8.73483657836914,
      "loss": 0.1844,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.0350141525268555,
      "rewards/margins": 6.067240238189697,
      "rewards/rejected": -13.102254867553711,
      "step": 790
    },
    {
      "epoch": 1.2542672160094173,
      "grad_norm": 2.8962628841400146,
      "learning_rate": 2.238678841830867e-06,
      "logits/chosen": 11.90459156036377,
      "logits/rejected": 11.352374076843262,
      "logps/chosen": -4.448246479034424,
      "logps/rejected": -8.658299446105957,
      "loss": 0.1638,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.67236852645874,
      "rewards/margins": 6.315079689025879,
      "rewards/rejected": -12.987447738647461,
      "step": 800
    },
    {
      "epoch": 1.2542672160094173,
      "eval_logits/chosen": 12.893868446350098,
      "eval_logits/rejected": 11.863032341003418,
      "eval_logps/chosen": -4.604159832000732,
      "eval_logps/rejected": -9.257932662963867,
      "eval_loss": 0.21881447732448578,
      "eval_rewards/accuracies": 0.9615384340286255,
      "eval_rewards/chosen": -6.906240463256836,
      "eval_rewards/margins": 6.980658531188965,
      "eval_rewards/rejected": -13.886898040771484,
      "eval_runtime": 14.6079,
      "eval_samples_per_second": 28.204,
      "eval_steps_per_second": 3.56,
      "step": 800
    },
    {
      "epoch": 1.2699627231704924,
      "grad_norm": 3.5829524993896484,
      "learning_rate": 2.186666916089239e-06,
      "logits/chosen": 12.411908149719238,
      "logits/rejected": 11.730254173278809,
      "logps/chosen": -4.65117883682251,
      "logps/rejected": -9.094849586486816,
      "loss": 0.1827,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.976768493652344,
      "rewards/margins": 6.665505886077881,
      "rewards/rejected": -13.642274856567383,
      "step": 810
    },
    {
      "epoch": 1.2856582303315676,
      "grad_norm": 2.836439371109009,
      "learning_rate": 2.134792428593971e-06,
      "logits/chosen": 12.173138618469238,
      "logits/rejected": 11.570637702941895,
      "logps/chosen": -4.847201824188232,
      "logps/rejected": -8.849002838134766,
      "loss": 0.1716,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.270802974700928,
      "rewards/margins": 6.002701282501221,
      "rewards/rejected": -13.273503303527832,
      "step": 820
    },
    {
      "epoch": 1.3013537374926427,
      "grad_norm": 3.5850038528442383,
      "learning_rate": 2.0830781332097446e-06,
      "logits/chosen": 12.883340835571289,
      "logits/rejected": 12.123892784118652,
      "logps/chosen": -4.7816948890686035,
      "logps/rejected": -9.729695320129395,
      "loss": 0.1652,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.172542572021484,
      "rewards/margins": 7.421999931335449,
      "rewards/rejected": -14.594541549682617,
      "step": 830
    },
    {
      "epoch": 1.3170492446537179,
      "grad_norm": 1.9497729539871216,
      "learning_rate": 2.031546713535688e-06,
      "logits/chosen": 12.579580307006836,
      "logits/rejected": 11.674839973449707,
      "logps/chosen": -4.67056131362915,
      "logps/rejected": -9.12755298614502,
      "loss": 0.1881,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.0058417320251465,
      "rewards/margins": 6.68548583984375,
      "rewards/rejected": -13.691328048706055,
      "step": 840
    },
    {
      "epoch": 1.332744751814793,
      "grad_norm": 2.5810794830322266,
      "learning_rate": 1.9802207729556023e-06,
      "logits/chosen": 12.369009017944336,
      "logits/rejected": 11.520233154296875,
      "logps/chosen": -4.780073165893555,
      "logps/rejected": -9.808363914489746,
      "loss": 0.2051,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.17011022567749,
      "rewards/margins": 7.542436122894287,
      "rewards/rejected": -14.712547302246094,
      "step": 850
    },
    {
      "epoch": 1.332744751814793,
      "eval_logits/chosen": 12.70075511932373,
      "eval_logits/rejected": 11.632950782775879,
      "eval_logps/chosen": -4.634037494659424,
      "eval_logps/rejected": -9.374123573303223,
      "eval_loss": 0.20189520716667175,
      "eval_rewards/accuracies": 0.9615384340286255,
      "eval_rewards/chosen": -6.951055526733398,
      "eval_rewards/margins": 7.110130310058594,
      "eval_rewards/rejected": -14.061185836791992,
      "eval_runtime": 14.6066,
      "eval_samples_per_second": 28.206,
      "eval_steps_per_second": 3.56,
      "step": 850
    },
    {
      "epoch": 1.3484402589758682,
      "grad_norm": 5.187539577484131,
      "learning_rate": 1.9291228247233607e-06,
      "logits/chosen": 12.224112510681152,
      "logits/rejected": 11.01191234588623,
      "logps/chosen": -4.748395919799805,
      "logps/rejected": -9.206069946289062,
      "loss": 0.2083,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.122593879699707,
      "rewards/margins": 6.686511039733887,
      "rewards/rejected": -13.809106826782227,
      "step": 860
    },
    {
      "epoch": 1.3641357661369433,
      "grad_norm": 1.1569608449935913,
      "learning_rate": 1.8782752820878636e-06,
      "logits/chosen": 11.579218864440918,
      "logits/rejected": 10.995697975158691,
      "logps/chosen": -4.387156009674072,
      "logps/rejected": -8.937141418457031,
      "loss": 0.1782,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.580733299255371,
      "rewards/margins": 6.824978828430176,
      "rewards/rejected": -13.405713081359863,
      "step": 870
    },
    {
      "epoch": 1.3798312732980185,
      "grad_norm": 5.9647111892700195,
      "learning_rate": 1.827700448461836e-06,
      "logits/chosen": 12.533722877502441,
      "logits/rejected": 12.121667861938477,
      "logps/chosen": -3.9464848041534424,
      "logps/rejected": -7.782477378845215,
      "loss": 0.237,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.919726848602295,
      "rewards/margins": 5.753990173339844,
      "rewards/rejected": -11.67371654510498,
      "step": 880
    },
    {
      "epoch": 1.3955267804590936,
      "grad_norm": 3.4284207820892334,
      "learning_rate": 1.7774205076388207e-06,
      "logits/chosen": 12.185909271240234,
      "logits/rejected": 11.781715393066406,
      "logps/chosen": -4.110888481140137,
      "logps/rejected": -8.276056289672852,
      "loss": 0.1904,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.166332721710205,
      "rewards/margins": 6.2477498054504395,
      "rewards/rejected": -12.414082527160645,
      "step": 890
    },
    {
      "epoch": 1.4112222876201688,
      "grad_norm": 2.537830352783203,
      "learning_rate": 1.7274575140626318e-06,
      "logits/chosen": 12.28978443145752,
      "logits/rejected": 11.305529594421387,
      "logps/chosen": -3.9210026264190674,
      "logps/rejected": -8.545848846435547,
      "loss": 0.1508,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.881503582000732,
      "rewards/margins": 6.937270164489746,
      "rewards/rejected": -12.818774223327637,
      "step": 900
    },
    {
      "epoch": 1.4112222876201688,
      "eval_logits/chosen": 12.542530059814453,
      "eval_logits/rejected": 11.450419425964355,
      "eval_logps/chosen": -4.722529888153076,
      "eval_logps/rejected": -9.546163558959961,
      "eval_loss": 0.18640650808811188,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -7.083794593811035,
      "eval_rewards/margins": 7.235450744628906,
      "eval_rewards/rejected": -14.319246292114258,
      "eval_runtime": 14.6107,
      "eval_samples_per_second": 28.199,
      "eval_steps_per_second": 3.559,
      "step": 900
    },
    {
      "epoch": 1.426917794781244,
      "grad_norm": 1.1588759422302246,
      "learning_rate": 1.677833383153542e-06,
      "logits/chosen": 12.432401657104492,
      "logits/rejected": 11.247361183166504,
      "logps/chosen": -4.893294334411621,
      "logps/rejected": -9.311574935913086,
      "loss": 0.1446,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.339941501617432,
      "rewards/margins": 6.627419948577881,
      "rewards/rejected": -13.967363357543945,
      "step": 910
    },
    {
      "epoch": 1.442613301942319,
      "grad_norm": 2.4712307453155518,
      "learning_rate": 1.6285698816954626e-06,
      "logits/chosen": 12.139798164367676,
      "logits/rejected": 11.248186111450195,
      "logps/chosen": -5.167681694030762,
      "logps/rejected": -10.253265380859375,
      "loss": 0.163,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.751523017883301,
      "rewards/margins": 7.6283769607543945,
      "rewards/rejected": -15.379899978637695,
      "step": 920
    },
    {
      "epoch": 1.4583088091033942,
      "grad_norm": 1.881538987159729,
      "learning_rate": 1.5796886182883053e-06,
      "logits/chosen": 11.9426908493042,
      "logits/rejected": 11.341384887695312,
      "logps/chosen": -5.030823230743408,
      "logps/rejected": -9.114843368530273,
      "loss": 0.1773,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.54623556137085,
      "rewards/margins": 6.126028060913086,
      "rewards/rejected": -13.672262191772461,
      "step": 930
    },
    {
      "epoch": 1.4740043162644694,
      "grad_norm": 5.751014709472656,
      "learning_rate": 1.5312110338697427e-06,
      "logits/chosen": 11.921320915222168,
      "logits/rejected": 11.005578994750977,
      "logps/chosen": -4.730319976806641,
      "logps/rejected": -9.198332786560059,
      "loss": 0.2058,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.095480442047119,
      "rewards/margins": 6.702019691467285,
      "rewards/rejected": -13.797500610351562,
      "step": 940
    },
    {
      "epoch": 1.4896998234255445,
      "grad_norm": 2.656139850616455,
      "learning_rate": 1.4831583923105e-06,
      "logits/chosen": 12.132013320922852,
      "logits/rejected": 11.254085540771484,
      "logps/chosen": -4.832255840301514,
      "logps/rejected": -9.473350524902344,
      "loss": 0.1827,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.24838399887085,
      "rewards/margins": 6.961642265319824,
      "rewards/rejected": -14.210027694702148,
      "step": 950
    },
    {
      "epoch": 1.4896998234255445,
      "eval_logits/chosen": 12.534186363220215,
      "eval_logits/rejected": 11.43524169921875,
      "eval_logps/chosen": -4.488946914672852,
      "eval_logps/rejected": -9.457845687866211,
      "eval_loss": 0.1744592934846878,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -6.733420372009277,
      "eval_rewards/margins": 7.4533467292785645,
      "eval_rewards/rejected": -14.186767578125,
      "eval_runtime": 14.6075,
      "eval_samples_per_second": 28.205,
      "eval_steps_per_second": 3.56,
      "step": 950
    },
    {
      "epoch": 1.5053953305866195,
      "grad_norm": 4.976431369781494,
      "learning_rate": 1.4355517710873184e-06,
      "logits/chosen": 12.053160667419434,
      "logits/rejected": 11.461877822875977,
      "logps/chosen": -4.3384108543396,
      "logps/rejected": -8.933584213256836,
      "loss": 0.1559,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.5076165199279785,
      "rewards/margins": 6.892758846282959,
      "rewards/rejected": -13.400375366210938,
      "step": 960
    },
    {
      "epoch": 1.5210908377476948,
      "grad_norm": 2.786695718765259,
      "learning_rate": 1.388412052037682e-06,
      "logits/chosen": 12.072885513305664,
      "logits/rejected": 10.973172187805176,
      "logps/chosen": -4.89219331741333,
      "logps/rejected": -9.545186996459961,
      "loss": 0.1334,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.338289737701416,
      "rewards/margins": 6.979489803314209,
      "rewards/rejected": -14.317779541015625,
      "step": 970
    },
    {
      "epoch": 1.5367863449087698,
      "grad_norm": 5.252014636993408,
      "learning_rate": 1.3417599122003464e-06,
      "logits/chosen": 12.37597370147705,
      "logits/rejected": 11.43901538848877,
      "logps/chosen": -5.274794101715088,
      "logps/rejected": -9.821117401123047,
      "loss": 0.1477,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.912191867828369,
      "rewards/margins": 6.819484710693359,
      "rewards/rejected": -14.73167610168457,
      "step": 980
    },
    {
      "epoch": 1.5524818520698451,
      "grad_norm": 2.243185520172119,
      "learning_rate": 1.2956158147457116e-06,
      "logits/chosen": 11.921972274780273,
      "logits/rejected": 11.30908489227295,
      "logps/chosen": -5.393095970153809,
      "logps/rejected": -10.215553283691406,
      "loss": 0.1561,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -8.089644432067871,
      "rewards/margins": 7.233686923980713,
      "rewards/rejected": -15.323330879211426,
      "step": 990
    },
    {
      "epoch": 1.56817735923092,
      "grad_norm": 3.4886062145233154,
      "learning_rate": 1.2500000000000007e-06,
      "logits/chosen": 11.753881454467773,
      "logits/rejected": 11.269731521606445,
      "logps/chosen": -5.015429496765137,
      "logps/rejected": -10.169350624084473,
      "loss": 0.1771,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.523144721984863,
      "rewards/margins": 7.7308807373046875,
      "rewards/rejected": -15.25402545928955,
      "step": 1000
    },
    {
      "epoch": 1.56817735923092,
      "eval_logits/chosen": 12.437786102294922,
      "eval_logits/rejected": 11.32490348815918,
      "eval_logps/chosen": -4.6446380615234375,
      "eval_logps/rejected": -9.707358360290527,
      "eval_loss": 0.16595196723937988,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -6.966956615447998,
      "eval_rewards/margins": 7.594080448150635,
      "eval_rewards/rejected": -14.561037063598633,
      "eval_runtime": 14.6109,
      "eval_samples_per_second": 28.198,
      "eval_steps_per_second": 3.559,
      "step": 1000
    },
    {
      "epoch": 1.5838728663919954,
      "grad_norm": 1.7607649564743042,
      "learning_rate": 1.204932476567175e-06,
      "logits/chosen": 11.258343696594238,
      "logits/rejected": 10.613565444946289,
      "logps/chosen": -4.758576393127441,
      "logps/rejected": -9.250303268432617,
      "loss": 0.1668,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.137864589691162,
      "rewards/margins": 6.7375898361206055,
      "rewards/rejected": -13.875455856323242,
      "step": 1010
    },
    {
      "epoch": 1.5995683735530704,
      "grad_norm": 3.2954118251800537,
      "learning_rate": 1.160433012552508e-06,
      "logits/chosen": 12.226592063903809,
      "logits/rejected": 11.355180740356445,
      "logps/chosen": -4.198899269104004,
      "logps/rejected": -8.675564765930176,
      "loss": 0.1824,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.298348903656006,
      "rewards/margins": 6.714995384216309,
      "rewards/rejected": -13.013345718383789,
      "step": 1020
    },
    {
      "epoch": 1.6152638807141457,
      "grad_norm": 2.833040237426758,
      "learning_rate": 1.11652112689164e-06,
      "logits/chosen": 11.937528610229492,
      "logits/rejected": 11.127592086791992,
      "logps/chosen": -4.438723087310791,
      "logps/rejected": -9.106756210327148,
      "loss": 0.1567,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.658083915710449,
      "rewards/margins": 7.002050876617432,
      "rewards/rejected": -13.660135269165039,
      "step": 1030
    },
    {
      "epoch": 1.6309593878752207,
      "grad_norm": 2.0544891357421875,
      "learning_rate": 1.073216080788921e-06,
      "logits/chosen": 12.242444038391113,
      "logits/rejected": 11.25694465637207,
      "logps/chosen": -4.525513172149658,
      "logps/rejected": -9.212661743164062,
      "loss": 0.1682,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.78826904296875,
      "rewards/margins": 7.030723571777344,
      "rewards/rejected": -13.818992614746094,
      "step": 1040
    },
    {
      "epoch": 1.6466548950362958,
      "grad_norm": 1.6990854740142822,
      "learning_rate": 1.0305368692688175e-06,
      "logits/chosen": 11.99101448059082,
      "logits/rejected": 11.161852836608887,
      "logps/chosen": -4.299143314361572,
      "logps/rejected": -9.032526016235352,
      "loss": 0.1143,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.448714256286621,
      "rewards/margins": 7.100072383880615,
      "rewards/rejected": -13.548787117004395,
      "step": 1050
    },
    {
      "epoch": 1.6466548950362958,
      "eval_logits/chosen": 12.486811637878418,
      "eval_logits/rejected": 11.354048728942871,
      "eval_logps/chosen": -4.389483451843262,
      "eval_logps/rejected": -9.478667259216309,
      "eval_loss": 0.16117088496685028,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -6.584225654602051,
      "eval_rewards/margins": 7.633774280548096,
      "eval_rewards/rejected": -14.217998504638672,
      "eval_runtime": 14.6006,
      "eval_samples_per_second": 28.218,
      "eval_steps_per_second": 3.561,
      "step": 1050
    },
    {
      "epoch": 1.662350402197371,
      "grad_norm": 3.3203821182250977,
      "learning_rate": 9.88502212844063e-07,
      "logits/chosen": 12.17997932434082,
      "logits/rejected": 11.155837059020996,
      "logps/chosen": -4.206795692443848,
      "logps/rejected": -9.341482162475586,
      "loss": 0.1631,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.310193061828613,
      "rewards/margins": 7.702028751373291,
      "rewards/rejected": -14.012222290039062,
      "step": 1060
    },
    {
      "epoch": 1.678045909358446,
      "grad_norm": 2.4238297939300537,
      "learning_rate": 9.471305493042243e-07,
      "logits/chosen": 11.849737167358398,
      "logits/rejected": 10.839106559753418,
      "logps/chosen": -4.497702598571777,
      "logps/rejected": -9.431967735290527,
      "loss": 0.1347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.746554374694824,
      "rewards/margins": 7.401397705078125,
      "rewards/rejected": -14.14795207977295,
      "step": 1070
    },
    {
      "epoch": 1.6937414165195213,
      "grad_norm": 1.895890712738037,
      "learning_rate": 9.064400256282757e-07,
      "logits/chosen": 12.01026725769043,
      "logits/rejected": 11.010753631591797,
      "logps/chosen": -4.6886305809021,
      "logps/rejected": -9.496674537658691,
      "loss": 0.1076,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.032945156097412,
      "rewards/margins": 7.212066650390625,
      "rewards/rejected": -14.245012283325195,
      "step": 1080
    },
    {
      "epoch": 1.7094369236805964,
      "grad_norm": 4.748386859893799,
      "learning_rate": 8.664484900247363e-07,
      "logits/chosen": 11.74091911315918,
      "logits/rejected": 11.233245849609375,
      "logps/chosen": -5.176927089691162,
      "logps/rejected": -9.68712043762207,
      "loss": 0.1532,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.765390872955322,
      "rewards/margins": 6.765290260314941,
      "rewards/rejected": -14.530679702758789,
      "step": 1090
    },
    {
      "epoch": 1.7251324308416716,
      "grad_norm": 2.932032346725464,
      "learning_rate": 8.271734841028553e-07,
      "logits/chosen": 11.53050708770752,
      "logits/rejected": 10.704401969909668,
      "logps/chosen": -5.003798484802246,
      "logps/rejected": -10.222925186157227,
      "loss": 0.1632,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.505698204040527,
      "rewards/margins": 7.8286895751953125,
      "rewards/rejected": -15.334388732910156,
      "step": 1100
    },
    {
      "epoch": 1.7251324308416716,
      "eval_logits/chosen": 12.376419067382812,
      "eval_logits/rejected": 11.235088348388672,
      "eval_logps/chosen": -4.780559539794922,
      "eval_logps/rejected": -9.94357967376709,
      "eval_loss": 0.1563359946012497,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -7.170839309692383,
      "eval_rewards/margins": 7.744529724121094,
      "eval_rewards/rejected": -14.915369033813477,
      "eval_runtime": 14.5952,
      "eval_samples_per_second": 28.229,
      "eval_steps_per_second": 3.563,
      "step": 1100
    },
    {
      "epoch": 1.7408279380027467,
      "grad_norm": 3.4405391216278076,
      "learning_rate": 7.886322351782782e-07,
      "logits/chosen": 12.490205764770508,
      "logits/rejected": 11.409021377563477,
      "logps/chosen": -4.385369300842285,
      "logps/rejected": -9.336843490600586,
      "loss": 0.1579,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.578053951263428,
      "rewards/margins": 7.427211761474609,
      "rewards/rejected": -14.005266189575195,
      "step": 1110
    },
    {
      "epoch": 1.7565234451638219,
      "grad_norm": 4.066647529602051,
      "learning_rate": 7.508416487165862e-07,
      "logits/chosen": 11.967781066894531,
      "logits/rejected": 11.07763671875,
      "logps/chosen": -4.826973915100098,
      "logps/rejected": -9.608440399169922,
      "loss": 0.1603,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.2404608726501465,
      "rewards/margins": 7.172199249267578,
      "rewards/rejected": -14.412660598754883,
      "step": 1120
    },
    {
      "epoch": 1.772218952324897,
      "grad_norm": 1.7111518383026123,
      "learning_rate": 7.138183009179922e-07,
      "logits/chosen": 12.116536140441895,
      "logits/rejected": 11.637681007385254,
      "logps/chosen": -4.51651668548584,
      "logps/rejected": -9.084489822387695,
      "loss": 0.1428,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.77477502822876,
      "rewards/margins": 6.8519606590271,
      "rewards/rejected": -13.626734733581543,
      "step": 1130
    },
    {
      "epoch": 1.7879144594859722,
      "grad_norm": 4.302910804748535,
      "learning_rate": 6.775784314464717e-07,
      "logits/chosen": 11.709549903869629,
      "logits/rejected": 11.382287979125977,
      "logps/chosen": -5.279540061950684,
      "logps/rejected": -9.892831802368164,
      "loss": 0.163,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.919310569763184,
      "rewards/margins": 6.919938564300537,
      "rewards/rejected": -14.839248657226562,
      "step": 1140
    },
    {
      "epoch": 1.8036099666470473,
      "grad_norm": 0.8747031092643738,
      "learning_rate": 6.421379363065142e-07,
      "logits/chosen": 12.214895248413086,
      "logits/rejected": 11.334197998046875,
      "logps/chosen": -4.599240303039551,
      "logps/rejected": -8.940774917602539,
      "loss": 0.1288,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.898859977722168,
      "rewards/margins": 6.512301445007324,
      "rewards/rejected": -13.411161422729492,
      "step": 1150
    },
    {
      "epoch": 1.8036099666470473,
      "eval_logits/chosen": 12.347259521484375,
      "eval_logits/rejected": 11.185004234313965,
      "eval_logps/chosen": -4.888545036315918,
      "eval_logps/rejected": -10.101993560791016,
      "eval_loss": 0.15526321530342102,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -7.332817077636719,
      "eval_rewards/margins": 7.8201727867126465,
      "eval_rewards/rejected": -15.15299129486084,
      "eval_runtime": 14.6049,
      "eval_samples_per_second": 28.21,
      "eval_steps_per_second": 3.56,
      "step": 1150
    },
    {
      "epoch": 1.8193054738081225,
      "grad_norm": 1.8233786821365356,
      "learning_rate": 6.075123608706093e-07,
      "logits/chosen": 11.725976943969727,
      "logits/rejected": 11.068318367004395,
      "logps/chosen": -5.016218185424805,
      "logps/rejected": -9.066827774047852,
      "loss": 0.1241,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.524327278137207,
      "rewards/margins": 6.075912952423096,
      "rewards/rejected": -13.600239753723145,
      "step": 1160
    },
    {
      "epoch": 1.8350009809691976,
      "grad_norm": 1.6913511753082275,
      "learning_rate": 5.737168930605272e-07,
      "logits/chosen": 11.813165664672852,
      "logits/rejected": 10.936023712158203,
      "logps/chosen": -5.16327428817749,
      "logps/rejected": -9.918088912963867,
      "loss": 0.1337,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.744911193847656,
      "rewards/margins": 7.132222652435303,
      "rewards/rejected": -14.877134323120117,
      "step": 1170
    },
    {
      "epoch": 1.8506964881302728,
      "grad_norm": 4.435967445373535,
      "learning_rate": 5.407663566854008e-07,
      "logits/chosen": 11.606057167053223,
      "logits/rejected": 10.97921085357666,
      "logps/chosen": -5.703640937805176,
      "logps/rejected": -10.870219230651855,
      "loss": 0.1245,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -8.555461883544922,
      "rewards/margins": 7.749865531921387,
      "rewards/rejected": -16.305328369140625,
      "step": 1180
    },
    {
      "epoch": 1.8663919952913477,
      "grad_norm": 2.9056365489959717,
      "learning_rate": 5.086752049395094e-07,
      "logits/chosen": 11.697361946105957,
      "logits/rejected": 10.928617477416992,
      "logps/chosen": -5.48267936706543,
      "logps/rejected": -10.689864158630371,
      "loss": 0.1251,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.224019050598145,
      "rewards/margins": 7.810777187347412,
      "rewards/rejected": -16.0347957611084,
      "step": 1190
    },
    {
      "epoch": 1.882087502452423,
      "grad_norm": 2.538482904434204,
      "learning_rate": 4.774575140626317e-07,
      "logits/chosen": 11.8221435546875,
      "logits/rejected": 11.30038070678711,
      "logps/chosen": -5.215182781219482,
      "logps/rejected": -9.782806396484375,
      "loss": 0.1543,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.822774410247803,
      "rewards/margins": 6.851436614990234,
      "rewards/rejected": -14.674212455749512,
      "step": 1200
    },
    {
      "epoch": 1.882087502452423,
      "eval_logits/chosen": 12.300190925598145,
      "eval_logits/rejected": 11.128490447998047,
      "eval_logps/chosen": -4.917614936828613,
      "eval_logps/rejected": -10.187230110168457,
      "eval_loss": 0.1518644094467163,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -7.37642240524292,
      "eval_rewards/margins": 7.904422283172607,
      "eval_rewards/rejected": -15.280843734741211,
      "eval_runtime": 14.6081,
      "eval_samples_per_second": 28.203,
      "eval_steps_per_second": 3.56,
      "step": 1200
    },
    {
      "epoch": 1.897783009613498,
      "grad_norm": 1.9161700010299683,
      "learning_rate": 4.4712697716573994e-07,
      "logits/chosen": 11.40344524383545,
      "logits/rejected": 10.652345657348633,
      "logps/chosen": -5.708310127258301,
      "logps/rejected": -10.750404357910156,
      "loss": 0.1319,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -8.562464714050293,
      "rewards/margins": 7.5631422996521,
      "rewards/rejected": -16.125606536865234,
      "step": 1210
    },
    {
      "epoch": 1.9134785167745734,
      "grad_norm": 2.1904044151306152,
      "learning_rate": 4.1769689822475147e-07,
      "logits/chosen": 11.742301940917969,
      "logits/rejected": 10.61946964263916,
      "logps/chosen": -4.744222640991211,
      "logps/rejected": -9.856466293334961,
      "loss": 0.1219,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.116333961486816,
      "rewards/margins": 7.668366432189941,
      "rewards/rejected": -14.784701347351074,
      "step": 1220
    },
    {
      "epoch": 1.9291740239356483,
      "grad_norm": 1.7852545976638794,
      "learning_rate": 3.891801862449629e-07,
      "logits/chosen": 11.89977741241455,
      "logits/rejected": 10.996416091918945,
      "logps/chosen": -4.8371357917785645,
      "logps/rejected": -9.81475830078125,
      "loss": 0.1179,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.255703926086426,
      "rewards/margins": 7.466433525085449,
      "rewards/rejected": -14.722137451171875,
      "step": 1230
    },
    {
      "epoch": 1.9448695310967237,
      "grad_norm": 3.7123889923095703,
      "learning_rate": 3.615893495987335e-07,
      "logits/chosen": 11.688365936279297,
      "logits/rejected": 10.34130573272705,
      "logps/chosen": -5.3013434410095215,
      "logps/rejected": -10.476262092590332,
      "loss": 0.1335,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.952016353607178,
      "rewards/margins": 7.7623772621154785,
      "rewards/rejected": -15.714391708374023,
      "step": 1240
    },
    {
      "epoch": 1.9605650382577986,
      "grad_norm": 3.9862449169158936,
      "learning_rate": 3.3493649053890325e-07,
      "logits/chosen": 12.021284103393555,
      "logits/rejected": 11.184505462646484,
      "logps/chosen": -4.448847770690918,
      "logps/rejected": -9.278366088867188,
      "loss": 0.1712,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.673272132873535,
      "rewards/margins": 7.244276523590088,
      "rewards/rejected": -13.917549133300781,
      "step": 1250
    },
    {
      "epoch": 1.9605650382577986,
      "eval_logits/chosen": 12.263810157775879,
      "eval_logits/rejected": 11.079519271850586,
      "eval_logps/chosen": -4.891733169555664,
      "eval_logps/rejected": -10.19001293182373,
      "eval_loss": 0.15064650774002075,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -7.337599754333496,
      "eval_rewards/margins": 7.947419166564941,
      "eval_rewards/rejected": -15.285017967224121,
      "eval_runtime": 14.5975,
      "eval_samples_per_second": 28.224,
      "eval_steps_per_second": 3.562,
      "step": 1250
    },
    {
      "epoch": 1.976260545418874,
      "grad_norm": 1.567117691040039,
      "learning_rate": 3.092332998903416e-07,
      "logits/chosen": 12.057940483093262,
      "logits/rejected": 11.069997787475586,
      "logps/chosen": -4.710134506225586,
      "logps/rejected": -9.204834938049316,
      "loss": 0.1332,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.065201759338379,
      "rewards/margins": 6.742051124572754,
      "rewards/rejected": -13.807252883911133,
      "step": 1260
    },
    {
      "epoch": 1.991956052579949,
      "grad_norm": 3.7871203422546387,
      "learning_rate": 2.844910519219632e-07,
      "logits/chosen": 11.46792984008789,
      "logits/rejected": 10.572579383850098,
      "logps/chosen": -5.138741970062256,
      "logps/rejected": -10.308446884155273,
      "loss": 0.1687,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.7081122398376465,
      "rewards/margins": 7.7545576095581055,
      "rewards/rejected": -15.462671279907227,
      "step": 1270
    },
    {
      "epoch": 2.00627820286443,
      "grad_norm": 3.568068265914917,
      "learning_rate": 2.6072059940146775e-07,
      "logits/chosen": 11.988948822021484,
      "logits/rejected": 11.214070320129395,
      "logps/chosen": -4.7047834396362305,
      "logps/rejected": -9.610939979553223,
      "loss": 0.1344,
      "rewards/accuracies": 0.9589040875434875,
      "rewards/chosen": -7.0571746826171875,
      "rewards/margins": 7.3592352867126465,
      "rewards/rejected": -14.416410446166992,
      "step": 1280
    },
    {
      "epoch": 2.021973710025505,
      "grad_norm": 4.052379608154297,
      "learning_rate": 2.3793236883495164e-07,
      "logits/chosen": 11.641098976135254,
      "logits/rejected": 11.267931938171387,
      "logps/chosen": -5.0538649559021,
      "logps/rejected": -9.780375480651855,
      "loss": 0.163,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.580796718597412,
      "rewards/margins": 7.089766502380371,
      "rewards/rejected": -14.670562744140625,
      "step": 1290
    },
    {
      "epoch": 2.0376692171865805,
      "grad_norm": 3.9337706565856934,
      "learning_rate": 2.1613635589349756e-07,
      "logits/chosen": 12.23253059387207,
      "logits/rejected": 11.380578994750977,
      "logps/chosen": -4.633624076843262,
      "logps/rejected": -9.106454849243164,
      "loss": 0.1749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.950437068939209,
      "rewards/margins": 6.709245204925537,
      "rewards/rejected": -13.65968132019043,
      "step": 1300
    },
    {
      "epoch": 2.0376692171865805,
      "eval_logits/chosen": 12.265684127807617,
      "eval_logits/rejected": 11.080260276794434,
      "eval_logps/chosen": -4.788154125213623,
      "eval_logps/rejected": -10.096734046936035,
      "eval_loss": 0.14751878380775452,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -7.18223237991333,
      "eval_rewards/margins": 7.962869644165039,
      "eval_rewards/rejected": -15.145101547241211,
      "eval_runtime": 14.6092,
      "eval_samples_per_second": 28.202,
      "eval_steps_per_second": 3.559,
      "step": 1300
    },
    {
      "epoch": 2.0533647243476554,
      "grad_norm": 3.6989145278930664,
      "learning_rate": 1.95342121028749e-07,
      "logits/chosen": 11.732830047607422,
      "logits/rejected": 11.045280456542969,
      "logps/chosen": -4.425314903259277,
      "logps/rejected": -9.951812744140625,
      "loss": 0.1298,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.637972831726074,
      "rewards/margins": 8.28974723815918,
      "rewards/rejected": -14.927721977233887,
      "step": 1310
    },
    {
      "epoch": 2.0690602315087308,
      "grad_norm": 1.380689263343811,
      "learning_rate": 1.7555878527937164e-07,
      "logits/chosen": 11.639039039611816,
      "logits/rejected": 10.867536544799805,
      "logps/chosen": -5.215638160705566,
      "logps/rejected": -10.307191848754883,
      "loss": 0.1054,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.82345724105835,
      "rewards/margins": 7.637331485748291,
      "rewards/rejected": -15.460787773132324,
      "step": 1320
    },
    {
      "epoch": 2.0847557386698057,
      "grad_norm": 2.8833718299865723,
      "learning_rate": 1.567950262702714e-07,
      "logits/chosen": 11.799827575683594,
      "logits/rejected": 11.053567886352539,
      "logps/chosen": -5.004476070404053,
      "logps/rejected": -9.889656066894531,
      "loss": 0.1195,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.506715297698975,
      "rewards/margins": 7.327768802642822,
      "rewards/rejected": -14.834482192993164,
      "step": 1330
    },
    {
      "epoch": 2.100451245830881,
      "grad_norm": 4.374328136444092,
      "learning_rate": 1.3905907440629752e-07,
      "logits/chosen": 11.815560340881348,
      "logits/rejected": 10.514938354492188,
      "logps/chosen": -5.09425687789917,
      "logps/rejected": -10.630200386047363,
      "loss": 0.1462,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.64138650894165,
      "rewards/margins": 8.303915023803711,
      "rewards/rejected": -15.945300102233887,
      "step": 1340
    },
    {
      "epoch": 2.116146752991956,
      "grad_norm": 5.649663925170898,
      "learning_rate": 1.223587092621162e-07,
      "logits/chosen": 11.845888137817383,
      "logits/rejected": 10.951406478881836,
      "logps/chosen": -5.54123592376709,
      "logps/rejected": -10.818798065185547,
      "loss": 0.151,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -8.311853408813477,
      "rewards/margins": 7.916342735290527,
      "rewards/rejected": -16.228199005126953,
      "step": 1350
    },
    {
      "epoch": 2.116146752991956,
      "eval_logits/chosen": 12.23206615447998,
      "eval_logits/rejected": 11.042497634887695,
      "eval_logps/chosen": -4.86654806137085,
      "eval_logps/rejected": -10.186172485351562,
      "eval_loss": 0.1482052505016327,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -7.2998223304748535,
      "eval_rewards/margins": 7.979436874389648,
      "eval_rewards/rejected": -15.27925968170166,
      "eval_runtime": 14.6066,
      "eval_samples_per_second": 28.206,
      "eval_steps_per_second": 3.56,
      "step": 1350
    },
    {
      "epoch": 2.1318422601530314,
      "grad_norm": 1.5727463960647583,
      "learning_rate": 1.067012561698319e-07,
      "logits/chosen": 11.984907150268555,
      "logits/rejected": 11.170934677124023,
      "logps/chosen": -5.278778076171875,
      "logps/rejected": -10.719067573547363,
      "loss": 0.1111,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.918168067932129,
      "rewards/margins": 8.160432815551758,
      "rewards/rejected": -16.078601837158203,
      "step": 1360
    },
    {
      "epoch": 2.1475377673141063,
      "grad_norm": 2.5720973014831543,
      "learning_rate": 9.209358300585474e-08,
      "logits/chosen": 11.944007873535156,
      "logits/rejected": 11.239874839782715,
      "logps/chosen": -4.983370304107666,
      "logps/rejected": -9.842253684997559,
      "loss": 0.125,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.475054740905762,
      "rewards/margins": 7.288325309753418,
      "rewards/rejected": -14.763381958007812,
      "step": 1370
    },
    {
      "epoch": 2.1632332744751817,
      "grad_norm": 1.8921817541122437,
      "learning_rate": 7.854209717842231e-08,
      "logits/chosen": 11.709084510803223,
      "logits/rejected": 10.745340347290039,
      "logps/chosen": -5.250277519226074,
      "logps/rejected": -11.172690391540527,
      "loss": 0.1323,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.8754167556762695,
      "rewards/margins": 8.883621215820312,
      "rewards/rejected": -16.759037017822266,
      "step": 1380
    },
    {
      "epoch": 2.1789287816362566,
      "grad_norm": 3.5800342559814453,
      "learning_rate": 6.605274281709929e-08,
      "logits/chosen": 11.36808967590332,
      "logits/rejected": 10.22724437713623,
      "logps/chosen": -5.402105808258057,
      "logps/rejected": -10.600914001464844,
      "loss": 0.1365,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -8.103157997131348,
      "rewards/margins": 7.798213005065918,
      "rewards/rejected": -15.901371955871582,
      "step": 1390
    },
    {
      "epoch": 2.194624288797332,
      "grad_norm": 3.021120548248291,
      "learning_rate": 5.463099816548578e-08,
      "logits/chosen": 12.123847007751465,
      "logits/rejected": 10.885251998901367,
      "logps/chosen": -4.701441287994385,
      "logps/rejected": -10.605305671691895,
      "loss": 0.1116,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.052162170410156,
      "rewards/margins": 8.855795860290527,
      "rewards/rejected": -15.907957077026367,
      "step": 1400
    },
    {
      "epoch": 2.194624288797332,
      "eval_logits/chosen": 12.214911460876465,
      "eval_logits/rejected": 11.020902633666992,
      "eval_logps/chosen": -4.897452354431152,
      "eval_logps/rejected": -10.230777740478516,
      "eval_loss": 0.149026557803154,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -7.3461785316467285,
      "eval_rewards/margins": 7.999988079071045,
      "eval_rewards/rejected": -15.346166610717773,
      "eval_runtime": 14.6104,
      "eval_samples_per_second": 28.199,
      "eval_steps_per_second": 3.559,
      "step": 1400
    },
    {
      "epoch": 2.210319795958407,
      "grad_norm": 1.5570991039276123,
      "learning_rate": 4.428187317827848e-08,
      "logits/chosen": 11.934289932250977,
      "logits/rejected": 11.13194465637207,
      "logps/chosen": -5.17191743850708,
      "logps/rejected": -10.771769523620605,
      "loss": 0.1247,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.757874965667725,
      "rewards/margins": 8.399779319763184,
      "rewards/rejected": -16.157655715942383,
      "step": 1410
    },
    {
      "epoch": 2.226015303119482,
      "grad_norm": 1.9914512634277344,
      "learning_rate": 3.5009907323737826e-08,
      "logits/chosen": 11.679354667663574,
      "logits/rejected": 10.788348197937012,
      "logps/chosen": -5.224305152893066,
      "logps/rejected": -10.575732231140137,
      "loss": 0.1292,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.836456298828125,
      "rewards/margins": 8.027141571044922,
      "rewards/rejected": -15.86359691619873,
      "step": 1420
    },
    {
      "epoch": 2.241710810280557,
      "grad_norm": 1.5110270977020264,
      "learning_rate": 2.681916759252917e-08,
      "logits/chosen": 11.340958595275879,
      "logits/rejected": 10.710260391235352,
      "logps/chosen": -5.453789710998535,
      "logps/rejected": -10.543638229370117,
      "loss": 0.1364,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -8.180685043334961,
      "rewards/margins": 7.634771823883057,
      "rewards/rejected": -15.815455436706543,
      "step": 1430
    },
    {
      "epoch": 2.2574063174416326,
      "grad_norm": 3.2789409160614014,
      "learning_rate": 1.9713246713805588e-08,
      "logits/chosen": 11.973535537719727,
      "logits/rejected": 11.382715225219727,
      "logps/chosen": -4.661351203918457,
      "logps/rejected": -10.070053100585938,
      "loss": 0.1372,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.992027282714844,
      "rewards/margins": 8.113051414489746,
      "rewards/rejected": -15.105077743530273,
      "step": 1440
    },
    {
      "epoch": 2.2731018246027075,
      "grad_norm": 3.0575263500213623,
      "learning_rate": 1.3695261579316776e-08,
      "logits/chosen": 11.72465991973877,
      "logits/rejected": 10.645977973937988,
      "logps/chosen": -5.164309978485107,
      "logps/rejected": -10.171631813049316,
      "loss": 0.1496,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.746464729309082,
      "rewards/margins": 7.510981559753418,
      "rewards/rejected": -15.2574462890625,
      "step": 1450
    },
    {
      "epoch": 2.2731018246027075,
      "eval_logits/chosen": 12.213281631469727,
      "eval_logits/rejected": 11.019485473632812,
      "eval_logps/chosen": -4.893021583557129,
      "eval_logps/rejected": -10.233271598815918,
      "eval_loss": 0.14757712185382843,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -7.33953332901001,
      "eval_rewards/margins": 8.010376930236816,
      "eval_rewards/rejected": -15.349907875061035,
      "eval_runtime": 14.6125,
      "eval_samples_per_second": 28.195,
      "eval_steps_per_second": 3.559,
      "step": 1450
    }
  ],
  "logging_steps": 10,
  "max_steps": 1500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 3.466434374075941e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}