{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.3961904761904762,
  "eval_steps": 50,
  "global_step": 1100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.012698412698412698,
      "grad_norm": 0.04716634005308151,
      "learning_rate": 4.999451708687114e-06,
      "logits/chosen": 14.883691787719727,
      "logits/rejected": 15.016583442687988,
      "logps/chosen": -0.29512909054756165,
      "logps/rejected": -0.30033987760543823,
      "loss": 1.0007,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.4426936209201813,
      "rewards/margins": 0.007816222496330738,
      "rewards/rejected": -0.45050984621047974,
      "step": 10
    },
    {
      "epoch": 0.025396825396825397,
      "grad_norm": 0.04392225295305252,
      "learning_rate": 4.997807075247147e-06,
      "logits/chosen": 14.705224990844727,
      "logits/rejected": 14.737253189086914,
      "logps/chosen": -0.3201700747013092,
      "logps/rejected": -0.280050128698349,
      "loss": 0.9984,
      "rewards/accuracies": 0.26249998807907104,
      "rewards/chosen": -0.4802550673484802,
      "rewards/margins": -0.060179851949214935,
      "rewards/rejected": -0.4200752377510071,
      "step": 20
    },
    {
      "epoch": 0.0380952380952381,
      "grad_norm": 0.04989476501941681,
      "learning_rate": 4.9950668210706795e-06,
      "logits/chosen": 15.147977828979492,
      "logits/rejected": 15.080279350280762,
      "logps/chosen": -0.3157380223274231,
      "logps/rejected": -0.2997627556324005,
      "loss": 0.9991,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": -0.47360706329345703,
      "rewards/margins": -0.023962898179888725,
      "rewards/rejected": -0.44964417815208435,
      "step": 30
    },
    {
      "epoch": 0.050793650793650794,
      "grad_norm": 0.054729390889406204,
      "learning_rate": 4.9912321481237616e-06,
      "logits/chosen": 15.301165580749512,
      "logits/rejected": 15.276555061340332,
      "logps/chosen": -0.30470195412635803,
      "logps/rejected": -0.29767152667045593,
      "loss": 0.9849,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.45705294609069824,
      "rewards/margins": -0.010545584373176098,
      "rewards/rejected": -0.44650736451148987,
      "step": 40
    },
    {
      "epoch": 0.06349206349206349,
      "grad_norm": 0.06035450100898743,
      "learning_rate": 4.986304738420684e-06,
      "logits/chosen": 14.600168228149414,
      "logits/rejected": 14.7944917678833,
      "logps/chosen": -0.32278841733932495,
      "logps/rejected": -0.3014402687549591,
      "loss": 0.9991,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.48418259620666504,
      "rewards/margins": -0.03202226758003235,
      "rewards/rejected": -0.4521603584289551,
      "step": 50
    },
    {
      "epoch": 0.06349206349206349,
      "eval_logits/chosen": 15.261337280273438,
      "eval_logits/rejected": 15.51547908782959,
      "eval_logps/chosen": -0.3022651970386505,
      "eval_logps/rejected": -0.3061661124229431,
      "eval_loss": 0.9846106171607971,
      "eval_rewards/accuracies": 0.40625,
      "eval_rewards/chosen": -0.4533977508544922,
      "eval_rewards/margins": 0.005851435009390116,
      "eval_rewards/rejected": -0.45924919843673706,
      "eval_runtime": 18.4033,
      "eval_samples_per_second": 27.712,
      "eval_steps_per_second": 3.478,
      "step": 50
    },
    {
      "epoch": 0.0761904761904762,
      "grad_norm": 0.052623528987169266,
      "learning_rate": 4.980286753286196e-06,
      "logits/chosen": 15.2726469039917,
      "logits/rejected": 15.245884895324707,
      "logps/chosen": -0.31394433975219727,
      "logps/rejected": -0.2805258333683014,
      "loss": 1.0063,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": -0.4709165096282959,
      "rewards/margins": -0.05012776702642441,
      "rewards/rejected": -0.4207887053489685,
      "step": 60
    },
    {
      "epoch": 0.08888888888888889,
      "grad_norm": 0.3512271046638489,
      "learning_rate": 4.973180832407471e-06,
      "logits/chosen": 15.302217483520508,
      "logits/rejected": 15.387911796569824,
      "logps/chosen": -0.30377697944641113,
      "logps/rejected": -0.29276782274246216,
      "loss": 0.9888,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.4556654393672943,
      "rewards/margins": -0.016513748094439507,
      "rewards/rejected": -0.43915170431137085,
      "step": 70
    },
    {
      "epoch": 0.10158730158730159,
      "grad_norm": 0.056646134704351425,
      "learning_rate": 4.964990092676263e-06,
      "logits/chosen": 15.843823432922363,
      "logits/rejected": 15.895855903625488,
      "logps/chosen": -0.3091468811035156,
      "logps/rejected": -0.27621737122535706,
      "loss": 0.9926,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -0.46372032165527344,
      "rewards/margins": -0.04939427971839905,
      "rewards/rejected": -0.4143260419368744,
      "step": 80
    },
    {
      "epoch": 0.11428571428571428,
      "grad_norm": 0.062348950654268265,
      "learning_rate": 4.9557181268217225e-06,
      "logits/chosen": 15.522564888000488,
      "logits/rejected": 15.576242446899414,
      "logps/chosen": -0.3092329502105713,
      "logps/rejected": -0.27670228481292725,
      "loss": 0.9812,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": -0.4638494551181793,
      "rewards/margins": -0.048796020448207855,
      "rewards/rejected": -0.41505345702171326,
      "step": 90
    },
    {
      "epoch": 0.12698412698412698,
      "grad_norm": 0.06779928505420685,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": 15.999185562133789,
      "logits/rejected": 15.802484512329102,
      "logps/chosen": -0.3173079490661621,
      "logps/rejected": -0.283602774143219,
      "loss": 0.981,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": -0.4759618639945984,
      "rewards/margins": -0.050557754933834076,
      "rewards/rejected": -0.4254041314125061,
      "step": 100
    },
    {
      "epoch": 0.12698412698412698,
      "eval_logits/chosen": 15.745854377746582,
      "eval_logits/rejected": 16.0076961517334,
      "eval_logps/chosen": -0.29866182804107666,
      "eval_logps/rejected": -0.3121251165866852,
      "eval_loss": 0.9764130115509033,
      "eval_rewards/accuracies": 0.453125,
      "eval_rewards/chosen": -0.4479926824569702,
      "eval_rewards/margins": 0.02019493840634823,
      "eval_rewards/rejected": -0.4681876599788666,
      "eval_runtime": 18.3383,
      "eval_samples_per_second": 27.811,
      "eval_steps_per_second": 3.49,
      "step": 100
    },
    {
      "epoch": 0.13968253968253969,
      "grad_norm": 0.07346926629543304,
      "learning_rate": 4.933947257182901e-06,
      "logits/chosen": 15.723541259765625,
      "logits/rejected": 15.764880180358887,
      "logps/chosen": -0.3212783634662628,
      "logps/rejected": -0.3159862160682678,
      "loss": 0.9859,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.4819175601005554,
      "rewards/margins": -0.00793826300650835,
      "rewards/rejected": -0.47397929430007935,
      "step": 110
    },
    {
      "epoch": 0.1523809523809524,
      "grad_norm": 0.06748441606760025,
      "learning_rate": 4.921457902821578e-06,
      "logits/chosen": 15.817390441894531,
      "logits/rejected": 16.023067474365234,
      "logps/chosen": -0.29811763763427734,
      "logps/rejected": -0.30654552578926086,
      "loss": 0.9827,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.447176456451416,
      "rewards/margins": 0.01264181174337864,
      "rewards/rejected": -0.4598182737827301,
      "step": 120
    },
    {
      "epoch": 0.16507936507936508,
      "grad_norm": 0.08042758703231812,
      "learning_rate": 4.907906416994146e-06,
      "logits/chosen": 15.45033073425293,
      "logits/rejected": 15.865328788757324,
      "logps/chosen": -0.27259278297424316,
      "logps/rejected": -0.31338179111480713,
      "loss": 0.9743,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.40888920426368713,
      "rewards/margins": 0.06118353083729744,
      "rewards/rejected": -0.4700726866722107,
      "step": 130
    },
    {
      "epoch": 0.17777777777777778,
      "grad_norm": 0.15669430792331696,
      "learning_rate": 4.893298743830168e-06,
      "logits/chosen": 16.17765235900879,
      "logits/rejected": 15.951130867004395,
      "logps/chosen": -0.3015133738517761,
      "logps/rejected": -0.2970424294471741,
      "loss": 0.9824,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.4522700905799866,
      "rewards/margins": -0.006706444080919027,
      "rewards/rejected": -0.44556355476379395,
      "step": 140
    },
    {
      "epoch": 0.19047619047619047,
      "grad_norm": 0.07115967571735382,
      "learning_rate": 4.8776412907378845e-06,
      "logits/chosen": 15.552728652954102,
      "logits/rejected": 15.788042068481445,
      "logps/chosen": -0.28624147176742554,
      "logps/rejected": -0.3222900331020355,
      "loss": 0.9727,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4293622076511383,
      "rewards/margins": 0.05407290905714035,
      "rewards/rejected": -0.48343515396118164,
      "step": 150
    },
    {
      "epoch": 0.19047619047619047,
      "eval_logits/chosen": 15.72143268585205,
      "eval_logits/rejected": 15.974818229675293,
      "eval_logps/chosen": -0.29440543055534363,
      "eval_logps/rejected": -0.33279579877853394,
      "eval_loss": 0.9540849328041077,
      "eval_rewards/accuracies": 0.546875,
      "eval_rewards/chosen": -0.44160816073417664,
      "eval_rewards/margins": 0.05758553743362427,
      "eval_rewards/rejected": -0.4991936683654785,
      "eval_runtime": 18.335,
      "eval_samples_per_second": 27.816,
      "eval_steps_per_second": 3.491,
      "step": 150
    },
    {
      "epoch": 0.20317460317460317,
      "grad_norm": 0.1144075095653534,
      "learning_rate": 4.860940925593703e-06,
      "logits/chosen": 15.63622760772705,
      "logits/rejected": 15.758941650390625,
      "logps/chosen": -0.310846745967865,
      "logps/rejected": -0.35185256600379944,
      "loss": 0.9517,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.4662701487541199,
      "rewards/margins": 0.061508744955062866,
      "rewards/rejected": -0.5277789235115051,
      "step": 160
    },
    {
      "epoch": 0.21587301587301588,
      "grad_norm": 0.1173442155122757,
      "learning_rate": 4.84320497372973e-06,
      "logits/chosen": 15.8660306930542,
      "logits/rejected": 15.83642292022705,
      "logps/chosen": -0.279310017824173,
      "logps/rejected": -0.3301998972892761,
      "loss": 0.9439,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.41896501183509827,
      "rewards/margins": 0.07633484899997711,
      "rewards/rejected": -0.49529990553855896,
      "step": 170
    },
    {
      "epoch": 0.22857142857142856,
      "grad_norm": 0.204214945435524,
      "learning_rate": 4.824441214720629e-06,
      "logits/chosen": 15.557527542114258,
      "logits/rejected": 15.321504592895508,
      "logps/chosen": -0.3029389977455139,
      "logps/rejected": -0.3163699507713318,
      "loss": 0.9369,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.4544084668159485,
      "rewards/margins": 0.020146463066339493,
      "rewards/rejected": -0.47455495595932007,
      "step": 180
    },
    {
      "epoch": 0.24126984126984127,
      "grad_norm": 0.14012843370437622,
      "learning_rate": 4.804657878971252e-06,
      "logits/chosen": 15.567469596862793,
      "logits/rejected": 15.717155456542969,
      "logps/chosen": -0.3400501310825348,
      "logps/rejected": -0.3475819230079651,
      "loss": 0.9443,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.5100752115249634,
      "rewards/margins": 0.011297682300209999,
      "rewards/rejected": -0.5213728547096252,
      "step": 190
    },
    {
      "epoch": 0.25396825396825395,
      "grad_norm": 0.12908300757408142,
      "learning_rate": 4.783863644106502e-06,
      "logits/chosen": 14.985162734985352,
      "logits/rejected": 15.223039627075195,
      "logps/chosen": -0.2876646816730499,
      "logps/rejected": -0.3777172267436981,
      "loss": 0.9034,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.4314970076084137,
      "rewards/margins": 0.13507887721061707,
      "rewards/rejected": -0.5665758848190308,
      "step": 200
    },
    {
      "epoch": 0.25396825396825395,
      "eval_logits/chosen": 15.242753028869629,
      "eval_logits/rejected": 15.523069381713867,
      "eval_logps/chosen": -0.2961229085922241,
      "eval_logps/rejected": -0.3966684341430664,
      "eval_loss": 0.9075753092765808,
      "eval_rewards/accuracies": 0.59375,
      "eval_rewards/chosen": -0.4441843330860138,
      "eval_rewards/margins": 0.1508183479309082,
      "eval_rewards/rejected": -0.5950026512145996,
      "eval_runtime": 18.341,
      "eval_samples_per_second": 27.807,
      "eval_steps_per_second": 3.489,
      "step": 200
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 0.13152211904525757,
      "learning_rate": 4.762067631165049e-06,
      "logits/chosen": 15.283185958862305,
      "logits/rejected": 15.545951843261719,
      "logps/chosen": -0.30045825242996216,
      "logps/rejected": -0.4519944190979004,
      "loss": 0.9085,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.45068734884262085,
      "rewards/margins": 0.22730426490306854,
      "rewards/rejected": -0.6779916286468506,
      "step": 210
    },
    {
      "epoch": 0.27936507936507937,
      "grad_norm": 0.14594660699367523,
      "learning_rate": 4.7392794005985324e-06,
      "logits/chosen": 15.53101921081543,
      "logits/rejected": 15.322232246398926,
      "logps/chosen": -0.3336530327796936,
      "logps/rejected": -0.36337023973464966,
      "loss": 0.9188,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.5004795789718628,
      "rewards/margins": 0.04457578808069229,
      "rewards/rejected": -0.5450553894042969,
      "step": 220
    },
    {
      "epoch": 0.2920634920634921,
      "grad_norm": 0.14854153990745544,
      "learning_rate": 4.715508948078037e-06,
      "logits/chosen": 14.157003402709961,
      "logits/rejected": 14.34089183807373,
      "logps/chosen": -0.28324443101882935,
      "logps/rejected": -0.41318759322166443,
      "loss": 0.9015,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.4248666763305664,
      "rewards/margins": 0.19491472840309143,
      "rewards/rejected": -0.6197813749313354,
      "step": 230
    },
    {
      "epoch": 0.3047619047619048,
      "grad_norm": 0.19532087445259094,
      "learning_rate": 4.690766700109659e-06,
      "logits/chosen": 14.729398727416992,
      "logits/rejected": 14.97203254699707,
      "logps/chosen": -0.3013172149658203,
      "logps/rejected": -0.41750937700271606,
      "loss": 0.8814,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.45197582244873047,
      "rewards/margins": 0.17428824305534363,
      "rewards/rejected": -0.6262640357017517,
      "step": 240
    },
    {
      "epoch": 0.31746031746031744,
      "grad_norm": 0.5795227885246277,
      "learning_rate": 4.665063509461098e-06,
      "logits/chosen": 14.328412055969238,
      "logits/rejected": 14.645428657531738,
      "logps/chosen": -0.3055194914340973,
      "logps/rejected": -0.509621262550354,
      "loss": 0.8478,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.45827922224998474,
      "rewards/margins": 0.30615273118019104,
      "rewards/rejected": -0.764431893825531,
      "step": 250
    },
    {
      "epoch": 0.31746031746031744,
      "eval_logits/chosen": 14.310781478881836,
      "eval_logits/rejected": 14.5652494430542,
      "eval_logps/chosen": -0.3155084252357483,
      "eval_logps/rejected": -0.8301784992218018,
      "eval_loss": 0.7700436115264893,
      "eval_rewards/accuracies": 0.640625,
      "eval_rewards/chosen": -0.47326260805130005,
      "eval_rewards/margins": 0.7720052003860474,
      "eval_rewards/rejected": -1.2452678680419922,
      "eval_runtime": 18.3413,
      "eval_samples_per_second": 27.806,
      "eval_steps_per_second": 3.489,
      "step": 250
    },
    {
      "epoch": 0.33015873015873015,
      "grad_norm": 0.24876689910888672,
      "learning_rate": 4.638410650401267e-06,
      "logits/chosen": 14.115270614624023,
      "logits/rejected": 14.167577743530273,
      "logps/chosen": -0.32328444719314575,
      "logps/rejected": -0.8564422726631165,
      "loss": 0.795,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.484926700592041,
      "rewards/margins": 0.7997367978096008,
      "rewards/rejected": -1.284663438796997,
      "step": 260
    },
    {
      "epoch": 0.34285714285714286,
      "grad_norm": 0.5779634118080139,
      "learning_rate": 4.610819813755038e-06,
      "logits/chosen": 14.019407272338867,
      "logits/rejected": 13.600751876831055,
      "logps/chosen": -0.35225847363471985,
      "logps/rejected": -1.1924464702606201,
      "loss": 0.7644,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5283876657485962,
      "rewards/margins": 1.2602821588516235,
      "rewards/rejected": -1.7886695861816406,
      "step": 270
    },
    {
      "epoch": 0.35555555555555557,
      "grad_norm": 0.2085695117712021,
      "learning_rate": 4.582303101775249e-06,
      "logits/chosen": 13.448820114135742,
      "logits/rejected": 13.464624404907227,
      "logps/chosen": -0.3442023992538452,
      "logps/rejected": -1.4783053398132324,
      "loss": 0.7671,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.516303539276123,
      "rewards/margins": 1.7011544704437256,
      "rewards/rejected": -2.2174580097198486,
      "step": 280
    },
    {
      "epoch": 0.3682539682539683,
      "grad_norm": 0.2825860381126404,
      "learning_rate": 4.55287302283426e-06,
      "logits/chosen": 13.038851737976074,
      "logits/rejected": 13.013893127441406,
      "logps/chosen": -0.3524690568447113,
      "logps/rejected": -1.2354861497879028,
      "loss": 0.7922,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5287035703659058,
      "rewards/margins": 1.3245255947113037,
      "rewards/rejected": -1.8532291650772095,
      "step": 290
    },
    {
      "epoch": 0.38095238095238093,
      "grad_norm": 0.30771639943122864,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": 13.67170524597168,
      "logits/rejected": 13.185449600219727,
      "logps/chosen": -0.3915162682533264,
      "logps/rejected": -1.3473880290985107,
      "loss": 0.7506,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.587274432182312,
      "rewards/margins": 1.433807611465454,
      "rewards/rejected": -2.0210821628570557,
      "step": 300
    },
    {
      "epoch": 0.38095238095238093,
      "eval_logits/chosen": 13.198552131652832,
      "eval_logits/rejected": 13.458879470825195,
      "eval_logps/chosen": -0.37132135033607483,
      "eval_logps/rejected": -1.4595866203308105,
      "eval_loss": 0.7391816973686218,
      "eval_rewards/accuracies": 0.640625,
      "eval_rewards/chosen": -0.5569820404052734,
      "eval_rewards/margins": 1.6323981285095215,
      "eval_rewards/rejected": -2.189380168914795,
      "eval_runtime": 18.3117,
      "eval_samples_per_second": 27.851,
      "eval_steps_per_second": 3.495,
      "step": 300
    },
    {
      "epoch": 0.39365079365079364,
      "grad_norm": 0.3021286129951477,
      "learning_rate": 4.491324795060491e-06,
      "logits/chosen": 13.347516059875488,
      "logits/rejected": 13.285726547241211,
      "logps/chosen": -0.4108191132545471,
      "logps/rejected": -1.436702013015747,
      "loss": 0.7134,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.6162286400794983,
      "rewards/margins": 1.5388243198394775,
      "rewards/rejected": -2.15505313873291,
      "step": 310
    },
    {
      "epoch": 0.40634920634920635,
      "grad_norm": 0.40123897790908813,
      "learning_rate": 4.4592336433146e-06,
      "logits/chosen": 12.949071884155273,
      "logits/rejected": 12.86626148223877,
      "logps/chosen": -0.42898029088974,
      "logps/rejected": -1.1741807460784912,
      "loss": 0.74,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.6434704065322876,
      "rewards/margins": 1.1178009510040283,
      "rewards/rejected": -1.7612712383270264,
      "step": 320
    },
    {
      "epoch": 0.41904761904761906,
      "grad_norm": 0.4613189697265625,
      "learning_rate": 4.426283106939474e-06,
      "logits/chosen": 12.796838760375977,
      "logits/rejected": 12.524165153503418,
      "logps/chosen": -0.48975634574890137,
      "logps/rejected": -1.2766364812850952,
      "loss": 0.7408,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7346345782279968,
      "rewards/margins": 1.180320143699646,
      "rewards/rejected": -1.9149547815322876,
      "step": 330
    },
    {
      "epoch": 0.43174603174603177,
      "grad_norm": 0.49141305685043335,
      "learning_rate": 4.3924876391293915e-06,
      "logits/chosen": 12.58080005645752,
      "logits/rejected": 12.5891695022583,
      "logps/chosen": -0.5303409099578857,
      "logps/rejected": -1.6262977123260498,
      "loss": 0.7501,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.7955113649368286,
      "rewards/margins": 1.643935203552246,
      "rewards/rejected": -2.4394466876983643,
      "step": 340
    },
    {
      "epoch": 0.4444444444444444,
      "grad_norm": 0.7533912062644958,
      "learning_rate": 4.357862063693486e-06,
      "logits/chosen": 12.15321159362793,
      "logits/rejected": 12.38386344909668,
      "logps/chosen": -0.5782582759857178,
      "logps/rejected": -1.580635905265808,
      "loss": 0.7405,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8673874139785767,
      "rewards/margins": 1.5035666227340698,
      "rewards/rejected": -2.3709540367126465,
      "step": 350
    },
    {
      "epoch": 0.4444444444444444,
      "eval_logits/chosen": 12.111034393310547,
      "eval_logits/rejected": 12.33246898651123,
      "eval_logps/chosen": -0.5941734910011292,
      "eval_logps/rejected": -1.6774102449417114,
      "eval_loss": 0.6953701972961426,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": -0.8912601470947266,
      "eval_rewards/margins": 1.6248550415039062,
      "eval_rewards/rejected": -2.516115188598633,
      "eval_runtime": 18.3366,
      "eval_samples_per_second": 27.813,
      "eval_steps_per_second": 3.49,
      "step": 350
    },
    {
      "epoch": 0.45714285714285713,
      "grad_norm": 1.168564796447754,
      "learning_rate": 4.322421568553529e-06,
      "logits/chosen": 11.706710815429688,
      "logits/rejected": 11.451011657714844,
      "logps/chosen": -0.758882462978363,
      "logps/rejected": -1.842167615890503,
      "loss": 0.702,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.1383236646652222,
      "rewards/margins": 1.6249277591705322,
      "rewards/rejected": -2.763251304626465,
      "step": 360
    },
    {
      "epoch": 0.46984126984126984,
      "grad_norm": 1.8962547779083252,
      "learning_rate": 4.286181699082008e-06,
      "logits/chosen": 11.41061019897461,
      "logits/rejected": 11.534398078918457,
      "logps/chosen": -1.1785552501678467,
      "logps/rejected": -2.235652446746826,
      "loss": 0.6781,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7678327560424805,
      "rewards/margins": 1.5856454372406006,
      "rewards/rejected": -3.3534786701202393,
      "step": 370
    },
    {
      "epoch": 0.48253968253968255,
      "grad_norm": 2.9152028560638428,
      "learning_rate": 4.249158351283414e-06,
      "logits/chosen": 11.582561492919922,
      "logits/rejected": 11.439531326293945,
      "logps/chosen": -1.9169971942901611,
      "logps/rejected": -2.788179397583008,
      "loss": 0.6033,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.875495672225952,
      "rewards/margins": 1.306773066520691,
      "rewards/rejected": -4.182269096374512,
      "step": 380
    },
    {
      "epoch": 0.49523809523809526,
      "grad_norm": 1.5070641040802002,
      "learning_rate": 4.211367764821722e-06,
      "logits/chosen": 11.340807914733887,
      "logits/rejected": 10.927321434020996,
      "logps/chosen": -2.1075730323791504,
      "logps/rejected": -3.4216556549072266,
      "loss": 0.5698,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.1613595485687256,
      "rewards/margins": 1.9711239337921143,
      "rewards/rejected": -5.13248348236084,
      "step": 390
    },
    {
      "epoch": 0.5079365079365079,
      "grad_norm": 2.6913516521453857,
      "learning_rate": 4.172826515897146e-06,
      "logits/chosen": 10.273612976074219,
      "logits/rejected": 10.116204261779785,
      "logps/chosen": -3.0047056674957275,
      "logps/rejected": -4.053646564483643,
      "loss": 0.5869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.507058143615723,
      "rewards/margins": 1.5734113454818726,
      "rewards/rejected": -6.080469608306885,
      "step": 400
    },
    {
      "epoch": 0.5079365079365079,
      "eval_logits/chosen": 9.58273696899414,
      "eval_logits/rejected": 9.765192031860352,
      "eval_logps/chosen": -2.4895575046539307,
      "eval_logps/rejected": -3.8314545154571533,
      "eval_loss": 0.5232856273651123,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": -3.7343361377716064,
      "eval_rewards/margins": 2.012845754623413,
      "eval_rewards/rejected": -5.7471818923950195,
      "eval_runtime": 18.3437,
      "eval_samples_per_second": 27.802,
      "eval_steps_per_second": 3.489,
      "step": 400
    },
    {
      "epoch": 0.5206349206349207,
      "grad_norm": 3.7197885513305664,
      "learning_rate": 4.133551509975264e-06,
      "logits/chosen": 9.081324577331543,
      "logits/rejected": 9.309822082519531,
      "logps/chosen": -2.2554564476013184,
      "logps/rejected": -3.2624363899230957,
      "loss": 0.5774,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.3831849098205566,
      "rewards/margins": 1.5104694366455078,
      "rewards/rejected": -4.893653869628906,
      "step": 410
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 2.757873773574829,
      "learning_rate": 4.093559974371725e-06,
      "logits/chosen": 8.145927429199219,
      "logits/rejected": 8.402036666870117,
      "logps/chosen": -3.1110777854919434,
      "logps/rejected": -4.202320098876953,
      "loss": 0.5084,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -4.666616916656494,
      "rewards/margins": 1.636863350868225,
      "rewards/rejected": -6.30348014831543,
      "step": 420
    },
    {
      "epoch": 0.546031746031746,
      "grad_norm": 3.3764102458953857,
      "learning_rate": 4.052869450695776e-06,
      "logits/chosen": 8.132574081420898,
      "logits/rejected": 8.386279106140137,
      "logps/chosen": -3.0871124267578125,
      "logps/rejected": -4.399088382720947,
      "loss": 0.5233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.630668640136719,
      "rewards/margins": 1.967963457107544,
      "rewards/rejected": -6.598631858825684,
      "step": 430
    },
    {
      "epoch": 0.5587301587301587,
      "grad_norm": 1.8188632726669312,
      "learning_rate": 4.011497787155938e-06,
      "logits/chosen": 8.104793548583984,
      "logits/rejected": 8.25890064239502,
      "logps/chosen": -3.598665237426758,
      "logps/rejected": -5.2809929847717285,
      "loss": 0.5272,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.397997856140137,
      "rewards/margins": 2.5234923362731934,
      "rewards/rejected": -7.921489715576172,
      "step": 440
    },
    {
      "epoch": 0.5714285714285714,
      "grad_norm": 2.528883934020996,
      "learning_rate": 3.969463130731183e-06,
      "logits/chosen": 8.675338745117188,
      "logits/rejected": 8.550267219543457,
      "logps/chosen": -3.582223892211914,
      "logps/rejected": -5.077345848083496,
      "loss": 0.5717,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -5.373335838317871,
      "rewards/margins": 2.2426836490631104,
      "rewards/rejected": -7.616019248962402,
      "step": 450
    },
    {
      "epoch": 0.5714285714285714,
      "eval_logits/chosen": 8.125189781188965,
      "eval_logits/rejected": 8.239119529724121,
      "eval_logps/chosen": -3.107839584350586,
      "eval_logps/rejected": -4.726868629455566,
      "eval_loss": 0.4568406939506531,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": -4.661759376525879,
      "eval_rewards/margins": 2.4285435676574707,
      "eval_rewards/rejected": -7.090303421020508,
      "eval_runtime": 18.3222,
      "eval_samples_per_second": 27.835,
      "eval_steps_per_second": 3.493,
      "step": 450
    },
    {
      "epoch": 0.5841269841269842,
      "grad_norm": 2.5079057216644287,
      "learning_rate": 3.92678391921108e-06,
      "logits/chosen": 8.503251075744629,
      "logits/rejected": 8.457250595092773,
      "logps/chosen": -3.499116897583008,
      "logps/rejected": -5.12787389755249,
      "loss": 0.4803,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.2486748695373535,
      "rewards/margins": 2.4431352615356445,
      "rewards/rejected": -7.69180965423584,
      "step": 460
    },
    {
      "epoch": 0.5968253968253968,
      "grad_norm": 1.869667887687683,
      "learning_rate": 3.88347887310836e-06,
      "logits/chosen": 7.972044467926025,
      "logits/rejected": 7.825163841247559,
      "logps/chosen": -3.6900856494903564,
      "logps/rejected": -5.166090488433838,
      "loss": 0.4733,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.535128593444824,
      "rewards/margins": 2.2140071392059326,
      "rewards/rejected": -7.749135494232178,
      "step": 470
    },
    {
      "epoch": 0.6095238095238096,
      "grad_norm": 2.8777735233306885,
      "learning_rate": 3.839566987447492e-06,
      "logits/chosen": 8.073511123657227,
      "logits/rejected": 8.345193862915039,
      "logps/chosen": -3.7006676197052,
      "logps/rejected": -5.271113395690918,
      "loss": 0.5048,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.551001071929932,
      "rewards/margins": 2.3556694984436035,
      "rewards/rejected": -7.906670570373535,
      "step": 480
    },
    {
      "epoch": 0.6222222222222222,
      "grad_norm": 2.7042839527130127,
      "learning_rate": 3.795067523432826e-06,
      "logits/chosen": 8.602261543273926,
      "logits/rejected": 8.32430362701416,
      "logps/chosen": -3.9612879753112793,
      "logps/rejected": -6.123431205749512,
      "loss": 0.4728,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.94193172454834,
      "rewards/margins": 3.2432150840759277,
      "rewards/rejected": -9.185147285461426,
      "step": 490
    },
    {
      "epoch": 0.6349206349206349,
      "grad_norm": 2.1625821590423584,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": 7.857954978942871,
      "logits/rejected": 7.460604190826416,
      "logps/chosen": -3.581023693084717,
      "logps/rejected": -5.477323532104492,
      "loss": 0.4296,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.371535778045654,
      "rewards/margins": 2.844449520111084,
      "rewards/rejected": -8.215986251831055,
      "step": 500
    },
    {
      "epoch": 0.6349206349206349,
      "eval_logits/chosen": 7.520671844482422,
      "eval_logits/rejected": 7.586723327636719,
      "eval_logps/chosen": -3.1659929752349854,
      "eval_logps/rejected": -4.955198764801025,
      "eval_loss": 0.4266711175441742,
      "eval_rewards/accuracies": 0.890625,
      "eval_rewards/chosen": -4.748989105224609,
      "eval_rewards/margins": 2.683809518814087,
      "eval_rewards/rejected": -7.432799339294434,
      "eval_runtime": 18.3456,
      "eval_samples_per_second": 27.8,
      "eval_steps_per_second": 3.489,
      "step": 500
    },
    {
      "epoch": 0.6476190476190476,
      "grad_norm": 4.2689104080200195,
      "learning_rate": 3.7043841852542884e-06,
      "logits/chosen": 7.948687553405762,
      "logits/rejected": 7.656899929046631,
      "logps/chosen": -2.8974719047546387,
      "logps/rejected": -4.501969337463379,
      "loss": 0.4981,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.346207618713379,
      "rewards/margins": 2.4067459106445312,
      "rewards/rejected": -6.752954006195068,
      "step": 510
    },
    {
      "epoch": 0.6603174603174603,
      "grad_norm": 3.299290895462036,
      "learning_rate": 3.658240087799655e-06,
      "logits/chosen": 7.676476955413818,
      "logits/rejected": 7.332755088806152,
      "logps/chosen": -3.9289379119873047,
      "logps/rejected": -5.498073577880859,
      "loss": 0.4736,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -5.893406867980957,
      "rewards/margins": 2.353703498840332,
      "rewards/rejected": -8.247110366821289,
      "step": 520
    },
    {
      "epoch": 0.6730158730158731,
      "grad_norm": 3.1058828830718994,
      "learning_rate": 3.611587947962319e-06,
      "logits/chosen": 7.516615867614746,
      "logits/rejected": 7.344383239746094,
      "logps/chosen": -3.57861328125,
      "logps/rejected": -5.48795223236084,
      "loss": 0.502,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.367920398712158,
      "rewards/margins": 2.8640081882476807,
      "rewards/rejected": -8.231928825378418,
      "step": 530
    },
    {
      "epoch": 0.6857142857142857,
      "grad_norm": 3.874868869781494,
      "learning_rate": 3.564448228912682e-06,
      "logits/chosen": 7.619947910308838,
      "logits/rejected": 7.048402309417725,
      "logps/chosen": -4.164804935455322,
      "logps/rejected": -5.771735191345215,
      "loss": 0.44,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.247206687927246,
      "rewards/margins": 2.4103951454162598,
      "rewards/rejected": -8.657602310180664,
      "step": 540
    },
    {
      "epoch": 0.6984126984126984,
      "grad_norm": 1.7583000659942627,
      "learning_rate": 3.516841607689501e-06,
      "logits/chosen": 7.068112850189209,
      "logits/rejected": 7.042977333068848,
      "logps/chosen": -3.760105609893799,
      "logps/rejected": -5.3382248878479,
      "loss": 0.4751,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.640158653259277,
      "rewards/margins": 2.3671793937683105,
      "rewards/rejected": -8.00733757019043,
      "step": 550
    },
    {
      "epoch": 0.6984126984126984,
      "eval_logits/chosen": 7.213573455810547,
      "eval_logits/rejected": 7.25667667388916,
      "eval_logps/chosen": -3.8712658882141113,
      "eval_logps/rejected": -5.882909774780273,
      "eval_loss": 0.3922204077243805,
      "eval_rewards/accuracies": 0.90625,
      "eval_rewards/chosen": -5.806899070739746,
      "eval_rewards/margins": 3.017465591430664,
      "eval_rewards/rejected": -8.824363708496094,
      "eval_runtime": 18.3231,
      "eval_samples_per_second": 27.834,
      "eval_steps_per_second": 3.493,
      "step": 550
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 1.9696826934814453,
      "learning_rate": 3.4687889661302577e-06,
      "logits/chosen": 7.829404354095459,
      "logits/rejected": 7.19503927230835,
      "logps/chosen": -4.589760780334473,
      "logps/rejected": -6.637518405914307,
      "loss": 0.4127,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.884641170501709,
      "rewards/margins": 3.0716373920440674,
      "rewards/rejected": -9.956277847290039,
      "step": 560
    },
    {
      "epoch": 0.7238095238095238,
      "grad_norm": 4.987663269042969,
      "learning_rate": 3.4203113817116955e-06,
      "logits/chosen": 7.03420877456665,
      "logits/rejected": 7.229029655456543,
      "logps/chosen": -3.7655014991760254,
      "logps/rejected": -6.0161333084106445,
      "loss": 0.4436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.648251533508301,
      "rewards/margins": 3.375948667526245,
      "rewards/rejected": -9.024200439453125,
      "step": 570
    },
    {
      "epoch": 0.7365079365079366,
      "grad_norm": 2.468047618865967,
      "learning_rate": 3.3714301183045382e-06,
      "logits/chosen": 6.352839946746826,
      "logits/rejected": 6.344137668609619,
      "logps/chosen": -2.9854183197021484,
      "logps/rejected": -5.105284690856934,
      "loss": 0.411,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.478127479553223,
      "rewards/margins": 3.1797995567321777,
      "rewards/rejected": -7.657926082611084,
      "step": 580
    },
    {
      "epoch": 0.7492063492063492,
      "grad_norm": 2.321638822555542,
      "learning_rate": 3.3221666168464584e-06,
      "logits/chosen": 7.339761257171631,
      "logits/rejected": 7.080865383148193,
      "logps/chosen": -3.7470335960388184,
      "logps/rejected": -5.2733564376831055,
      "loss": 0.4297,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.62054967880249,
      "rewards/margins": 2.2894842624664307,
      "rewards/rejected": -7.9100341796875,
      "step": 590
    },
    {
      "epoch": 0.7619047619047619,
      "grad_norm": 6.850190162658691,
      "learning_rate": 3.272542485937369e-06,
      "logits/chosen": 7.278192043304443,
      "logits/rejected": 6.769750118255615,
      "logps/chosen": -3.9920971393585205,
      "logps/rejected": -5.948962211608887,
      "loss": 0.4537,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.988145351409912,
      "rewards/margins": 2.9352970123291016,
      "rewards/rejected": -8.923442840576172,
      "step": 600
    },
    {
      "epoch": 0.7619047619047619,
      "eval_logits/chosen": 6.745523929595947,
      "eval_logits/rejected": 6.732550144195557,
      "eval_logps/chosen": -3.667492628097534,
      "eval_logps/rejected": -5.859607696533203,
      "eval_loss": 0.368425577878952,
      "eval_rewards/accuracies": 0.921875,
      "eval_rewards/chosen": -5.501238822937012,
      "eval_rewards/margins": 3.288173198699951,
      "eval_rewards/rejected": -8.789411544799805,
      "eval_runtime": 18.3412,
      "eval_samples_per_second": 27.806,
      "eval_steps_per_second": 3.489,
      "step": 600
    },
    {
      "epoch": 0.7746031746031746,
      "grad_norm": 3.103388786315918,
      "learning_rate": 3.222579492361179e-06,
      "logits/chosen": 6.413697719573975,
      "logits/rejected": 6.046789646148682,
      "logps/chosen": -4.207709312438965,
      "logps/rejected": -6.105495452880859,
      "loss": 0.4315,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.311563968658447,
      "rewards/margins": 2.846680164337158,
      "rewards/rejected": -9.158244132995605,
      "step": 610
    },
    {
      "epoch": 0.7873015873015873,
      "grad_norm": 4.581955432891846,
      "learning_rate": 3.1722995515381644e-06,
      "logits/chosen": 6.37155818939209,
      "logits/rejected": 6.382074356079102,
      "logps/chosen": -4.053309917449951,
      "logps/rejected": -5.929131507873535,
      "loss": 0.4161,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.079965114593506,
      "rewards/margins": 2.813732624053955,
      "rewards/rejected": -8.893697738647461,
      "step": 620
    },
    {
      "epoch": 0.8,
      "grad_norm": 3.6864757537841797,
      "learning_rate": 3.121724717912138e-06,
      "logits/chosen": 5.701592445373535,
      "logits/rejected": 5.8575944900512695,
      "logps/chosen": -3.532477617263794,
      "logps/rejected": -5.849920749664307,
      "loss": 0.3621,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.2987165451049805,
      "rewards/margins": 3.4761645793914795,
      "rewards/rejected": -8.774881362915039,
      "step": 630
    },
    {
      "epoch": 0.8126984126984127,
      "grad_norm": 3.3179032802581787,
      "learning_rate": 3.0708771752766397e-06,
      "logits/chosen": 5.983328819274902,
      "logits/rejected": 5.853272914886475,
      "logps/chosen": -3.7127442359924316,
      "logps/rejected": -5.6633429527282715,
      "loss": 0.4325,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.569116115570068,
      "rewards/margins": 2.9258980751037598,
      "rewards/rejected": -8.495015144348145,
      "step": 640
    },
    {
      "epoch": 0.8253968253968254,
      "grad_norm": 3.3010666370391846,
      "learning_rate": 3.019779227044398e-06,
      "logits/chosen": 7.427163600921631,
      "logits/rejected": 7.144589900970459,
      "logps/chosen": -3.395155429840088,
      "logps/rejected": -5.485442638397217,
      "loss": 0.4376,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.0927324295043945,
      "rewards/margins": 3.1354317665100098,
      "rewards/rejected": -8.228163719177246,
      "step": 650
    },
    {
      "epoch": 0.8253968253968254,
      "eval_logits/chosen": 6.977372169494629,
      "eval_logits/rejected": 6.941837787628174,
      "eval_logps/chosen": -3.4915952682495117,
      "eval_logps/rejected": -5.880876541137695,
      "eval_loss": 0.3507956266403198,
      "eval_rewards/accuracies": 0.921875,
      "eval_rewards/chosen": -5.237393379211426,
      "eval_rewards/margins": 3.5839221477508545,
      "eval_rewards/rejected": -8.821314811706543,
      "eval_runtime": 18.3366,
      "eval_samples_per_second": 27.813,
      "eval_steps_per_second": 3.49,
      "step": 650
    },
    {
      "epoch": 0.8380952380952381,
      "grad_norm": 6.439872741699219,
      "learning_rate": 2.9684532864643123e-06,
      "logits/chosen": 7.413880825042725,
      "logits/rejected": 7.040737152099609,
      "logps/chosen": -4.771336078643799,
      "logps/rejected": -7.235314846038818,
      "loss": 0.381,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -7.157004356384277,
      "rewards/margins": 3.6959691047668457,
      "rewards/rejected": -10.852972984313965,
      "step": 660
    },
    {
      "epoch": 0.8507936507936508,
      "grad_norm": 5.114463806152344,
      "learning_rate": 2.9169218667902562e-06,
      "logits/chosen": 6.7717437744140625,
      "logits/rejected": 6.568068504333496,
      "logps/chosen": -3.372582197189331,
      "logps/rejected": -5.318942070007324,
      "loss": 0.4385,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.058873653411865,
      "rewards/margins": 2.9195384979248047,
      "rewards/rejected": -7.9784135818481445,
      "step": 670
    },
    {
      "epoch": 0.8634920634920635,
      "grad_norm": 2.7731258869171143,
      "learning_rate": 2.8652075714060296e-06,
      "logits/chosen": 6.759649753570557,
      "logits/rejected": 6.250086784362793,
      "logps/chosen": -3.9128894805908203,
      "logps/rejected": -6.746951103210449,
      "loss": 0.3879,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.8693342208862305,
      "rewards/margins": 4.251092433929443,
      "rewards/rejected": -10.120426177978516,
      "step": 680
    },
    {
      "epoch": 0.8761904761904762,
      "grad_norm": 2.517974615097046,
      "learning_rate": 2.813333083910761e-06,
      "logits/chosen": 7.3841047286987305,
      "logits/rejected": 7.0824761390686035,
      "logps/chosen": -4.643451690673828,
      "logps/rejected": -6.954865455627441,
      "loss": 0.3451,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.965176582336426,
      "rewards/margins": 3.467121124267578,
      "rewards/rejected": -10.43229866027832,
      "step": 690
    },
    {
      "epoch": 0.8888888888888888,
      "grad_norm": 4.514930725097656,
      "learning_rate": 2.761321158169134e-06,
      "logits/chosen": 6.828726291656494,
      "logits/rejected": 6.3069562911987305,
      "logps/chosen": -4.702274322509766,
      "logps/rejected": -7.075028896331787,
      "loss": 0.4094,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -7.053411960601807,
      "rewards/margins": 3.5591323375701904,
      "rewards/rejected": -10.612543106079102,
      "step": 700
    },
    {
      "epoch": 0.8888888888888888,
      "eval_logits/chosen": 6.324803352355957,
      "eval_logits/rejected": 6.258608818054199,
      "eval_logps/chosen": -3.811472177505493,
      "eval_logps/rejected": -6.270685195922852,
      "eval_loss": 0.33448922634124756,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -5.717207431793213,
      "eval_rewards/margins": 3.6888198852539062,
      "eval_rewards/rejected": -9.406026840209961,
      "eval_runtime": 18.3448,
      "eval_samples_per_second": 27.801,
      "eval_steps_per_second": 3.489,
      "step": 700
    },
    {
      "epoch": 0.9015873015873016,
      "grad_norm": 2.871471405029297,
      "learning_rate": 2.70919460833079e-06,
      "logits/chosen": 6.418933868408203,
      "logits/rejected": 6.141990661621094,
      "logps/chosen": -4.123613357543945,
      "logps/rejected": -6.900608062744141,
      "loss": 0.3604,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.18541955947876,
      "rewards/margins": 4.165491580963135,
      "rewards/rejected": -10.350912094116211,
      "step": 710
    },
    {
      "epoch": 0.9142857142857143,
      "grad_norm": 5.5512824058532715,
      "learning_rate": 2.6569762988232838e-06,
      "logits/chosen": 6.1132917404174805,
      "logits/rejected": 6.01393461227417,
      "logps/chosen": -4.065993309020996,
      "logps/rejected": -6.6075005531311035,
      "loss": 0.3682,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.098989963531494,
      "rewards/margins": 3.812260150909424,
      "rewards/rejected": -9.911249160766602,
      "step": 720
    },
    {
      "epoch": 0.926984126984127,
      "grad_norm": 2.5356554985046387,
      "learning_rate": 2.604689134322999e-06,
      "logits/chosen": 6.585450649261475,
      "logits/rejected": 5.800690650939941,
      "logps/chosen": -4.80672550201416,
      "logps/rejected": -6.856999397277832,
      "loss": 0.3886,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -7.210087776184082,
      "rewards/margins": 3.075411319732666,
      "rewards/rejected": -10.28549861907959,
      "step": 730
    },
    {
      "epoch": 0.9396825396825397,
      "grad_norm": 3.685417413711548,
      "learning_rate": 2.5523560497083927e-06,
      "logits/chosen": 5.728347301483154,
      "logits/rejected": 5.568971157073975,
      "logps/chosen": -4.184335231781006,
      "logps/rejected": -6.79797887802124,
      "loss": 0.3548,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.27650260925293,
      "rewards/margins": 3.9204659461975098,
      "rewards/rejected": -10.196969032287598,
      "step": 740
    },
    {
      "epoch": 0.9523809523809523,
      "grad_norm": 3.107419490814209,
      "learning_rate": 2.5e-06,
      "logits/chosen": 6.063660621643066,
      "logits/rejected": 5.694943428039551,
      "logps/chosen": -3.863433837890625,
      "logps/rejected": -6.4726691246032715,
      "loss": 0.3706,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.7951507568359375,
      "rewards/margins": 3.9138541221618652,
      "rewards/rejected": -9.709004402160645,
      "step": 750
    },
    {
      "epoch": 0.9523809523809523,
      "eval_logits/chosen": 6.257328033447266,
      "eval_logits/rejected": 6.1753411293029785,
      "eval_logps/chosen": -3.6567726135253906,
      "eval_logps/rejected": -6.332419395446777,
      "eval_loss": 0.31834185123443604,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -5.485158920288086,
      "eval_rewards/margins": 4.013469696044922,
      "eval_rewards/rejected": -9.498628616333008,
      "eval_runtime": 18.3371,
      "eval_samples_per_second": 27.812,
      "eval_steps_per_second": 3.49,
      "step": 750
    },
    {
      "epoch": 0.9650793650793651,
      "grad_norm": 5.022021293640137,
      "learning_rate": 2.447643950291608e-06,
      "logits/chosen": 6.415736198425293,
      "logits/rejected": 6.128665924072266,
      "logps/chosen": -3.9557278156280518,
      "logps/rejected": -6.381206035614014,
      "loss": 0.3938,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.933591365814209,
      "rewards/margins": 3.6382179260253906,
      "rewards/rejected": -9.571809768676758,
      "step": 760
    },
    {
      "epoch": 0.9777777777777777,
      "grad_norm": 4.283695220947266,
      "learning_rate": 2.3953108656770018e-06,
      "logits/chosen": 5.191827774047852,
      "logits/rejected": 5.130216598510742,
      "logps/chosen": -3.842036724090576,
      "logps/rejected": -6.3525071144104,
      "loss": 0.3744,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.763054847717285,
      "rewards/margins": 3.7657058238983154,
      "rewards/rejected": -9.52876091003418,
      "step": 770
    },
    {
      "epoch": 0.9904761904761905,
      "grad_norm": 4.412943363189697,
      "learning_rate": 2.3430237011767166e-06,
      "logits/chosen": 6.2709760665893555,
      "logits/rejected": 5.860433101654053,
      "logps/chosen": -4.709076881408691,
      "logps/rejected": -7.461434841156006,
      "loss": 0.3914,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.063615322113037,
      "rewards/margins": 4.128536224365234,
      "rewards/rejected": -11.19215202331543,
      "step": 780
    },
    {
      "epoch": 1.0025396825396826,
      "grad_norm": 2.1262686252593994,
      "learning_rate": 2.290805391669212e-06,
      "logits/chosen": 6.877432346343994,
      "logits/rejected": 6.122270584106445,
      "logps/chosen": -4.66621732711792,
      "logps/rejected": -7.71566915512085,
      "loss": 0.3467,
      "rewards/accuracies": 0.8947368264198303,
      "rewards/chosen": -6.999325275421143,
      "rewards/margins": 4.574176788330078,
      "rewards/rejected": -11.573503494262695,
      "step": 790
    },
    {
      "epoch": 1.0152380952380953,
      "grad_norm": 4.042492389678955,
      "learning_rate": 2.238678841830867e-06,
      "logits/chosen": 6.55672550201416,
      "logits/rejected": 5.752241134643555,
      "logps/chosen": -4.66592264175415,
      "logps/rejected": -6.966067314147949,
      "loss": 0.3404,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.998884677886963,
      "rewards/margins": 3.4502170085906982,
      "rewards/rejected": -10.449102401733398,
      "step": 800
    },
    {
      "epoch": 1.0152380952380953,
      "eval_logits/chosen": 6.143824100494385,
      "eval_logits/rejected": 6.0408477783203125,
      "eval_logps/chosen": -3.900071144104004,
      "eval_logps/rejected": -6.64994478225708,
      "eval_loss": 0.30234017968177795,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -5.850106239318848,
      "eval_rewards/margins": 4.124810695648193,
      "eval_rewards/rejected": -9.974918365478516,
      "eval_runtime": 18.317,
      "eval_samples_per_second": 27.843,
      "eval_steps_per_second": 3.494,
      "step": 800
    },
    {
      "epoch": 1.027936507936508,
      "grad_norm": 4.464768886566162,
      "learning_rate": 2.186666916089239e-06,
      "logits/chosen": 5.800255298614502,
      "logits/rejected": 5.524979114532471,
      "logps/chosen": -4.174003601074219,
      "logps/rejected": -6.4187140464782715,
      "loss": 0.3517,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.261005401611328,
      "rewards/margins": 3.367064952850342,
      "rewards/rejected": -9.628071784973145,
      "step": 810
    },
    {
      "epoch": 1.0406349206349206,
      "grad_norm": 2.864623546600342,
      "learning_rate": 2.134792428593971e-06,
      "logits/chosen": 5.549002170562744,
      "logits/rejected": 5.762969017028809,
      "logps/chosen": -4.055948734283447,
      "logps/rejected": -6.332015037536621,
      "loss": 0.3525,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.083922386169434,
      "rewards/margins": 3.4140994548797607,
      "rewards/rejected": -9.498022079467773,
      "step": 820
    },
    {
      "epoch": 1.0533333333333332,
      "grad_norm": 4.997330188751221,
      "learning_rate": 2.0830781332097446e-06,
      "logits/chosen": 5.439722061157227,
      "logits/rejected": 5.755289554595947,
      "logps/chosen": -4.239761829376221,
      "logps/rejected": -6.699388027191162,
      "loss": 0.3423,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.359642028808594,
      "rewards/margins": 3.689438581466675,
      "rewards/rejected": -10.049081802368164,
      "step": 830
    },
    {
      "epoch": 1.066031746031746,
      "grad_norm": 4.705864906311035,
      "learning_rate": 2.031546713535688e-06,
      "logits/chosen": 6.309781074523926,
      "logits/rejected": 6.002473831176758,
      "logps/chosen": -3.9887123107910156,
      "logps/rejected": -6.414917945861816,
      "loss": 0.327,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.983067989349365,
      "rewards/margins": 3.639307737350464,
      "rewards/rejected": -9.622376441955566,
      "step": 840
    },
    {
      "epoch": 1.0787301587301588,
      "grad_norm": 4.2711286544799805,
      "learning_rate": 1.9802207729556023e-06,
      "logits/chosen": 6.275202751159668,
      "logits/rejected": 5.929355621337891,
      "logps/chosen": -4.061328887939453,
      "logps/rejected": -6.566980838775635,
      "loss": 0.3587,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.09199333190918,
      "rewards/margins": 3.758476734161377,
      "rewards/rejected": -9.850470542907715,
      "step": 850
    },
    {
      "epoch": 1.0787301587301588,
      "eval_logits/chosen": 5.9675092697143555,
      "eval_logits/rejected": 5.841681480407715,
      "eval_logps/chosen": -3.5920660495758057,
      "eval_logps/rejected": -6.45810604095459,
      "eval_loss": 0.2958359122276306,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -5.38809871673584,
      "eval_rewards/margins": 4.2990593910217285,
      "eval_rewards/rejected": -9.687158584594727,
      "eval_runtime": 18.3401,
      "eval_samples_per_second": 27.808,
      "eval_steps_per_second": 3.49,
      "step": 850
    },
    {
      "epoch": 1.0914285714285714,
      "grad_norm": 2.666412115097046,
      "learning_rate": 1.9291228247233607e-06,
      "logits/chosen": 6.118748188018799,
      "logits/rejected": 5.885035991668701,
      "logps/chosen": -3.863534927368164,
      "logps/rejected": -6.175184726715088,
      "loss": 0.3297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.795302391052246,
      "rewards/margins": 3.467475175857544,
      "rewards/rejected": -9.262777328491211,
      "step": 860
    },
    {
      "epoch": 1.104126984126984,
      "grad_norm": 2.253833293914795,
      "learning_rate": 1.8782752820878636e-06,
      "logits/chosen": 6.99627685546875,
      "logits/rejected": 6.862641334533691,
      "logps/chosen": -3.3962769508361816,
      "logps/rejected": -5.6661272048950195,
      "loss": 0.339,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.094415187835693,
      "rewards/margins": 3.404775619506836,
      "rewards/rejected": -8.499190330505371,
      "step": 870
    },
    {
      "epoch": 1.116825396825397,
      "grad_norm": 4.063273906707764,
      "learning_rate": 1.827700448461836e-06,
      "logits/chosen": 5.741837024688721,
      "logits/rejected": 5.7893548011779785,
      "logps/chosen": -3.724289655685425,
      "logps/rejected": -6.367809772491455,
      "loss": 0.304,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.586434841156006,
      "rewards/margins": 3.965280532836914,
      "rewards/rejected": -9.551714897155762,
      "step": 880
    },
    {
      "epoch": 1.1295238095238096,
      "grad_norm": 4.975926399230957,
      "learning_rate": 1.7774205076388207e-06,
      "logits/chosen": 6.33252477645874,
      "logits/rejected": 6.018566131591797,
      "logps/chosen": -4.238375186920166,
      "logps/rejected": -7.243282318115234,
      "loss": 0.3124,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.35756254196167,
      "rewards/margins": 4.50736141204834,
      "rewards/rejected": -10.864924430847168,
      "step": 890
    },
    {
      "epoch": 1.1422222222222222,
      "grad_norm": 3.159183979034424,
      "learning_rate": 1.7274575140626318e-06,
      "logits/chosen": 6.32235860824585,
      "logits/rejected": 5.313261985778809,
      "logps/chosen": -4.42888879776001,
      "logps/rejected": -7.219472408294678,
      "loss": 0.3122,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.643332481384277,
      "rewards/margins": 4.185876846313477,
      "rewards/rejected": -10.82921028137207,
      "step": 900
    },
    {
      "epoch": 1.1422222222222222,
      "eval_logits/chosen": 5.802257537841797,
      "eval_logits/rejected": 5.6801910400390625,
      "eval_logps/chosen": -3.7256534099578857,
      "eval_logps/rejected": -6.689548492431641,
      "eval_loss": 0.28521302342414856,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -5.588479995727539,
      "eval_rewards/margins": 4.445842742919922,
      "eval_rewards/rejected": -10.034323692321777,
      "eval_runtime": 18.2302,
      "eval_samples_per_second": 27.976,
      "eval_steps_per_second": 3.511,
      "step": 900
    },
    {
      "epoch": 1.154920634920635,
      "grad_norm": 5.580939292907715,
      "learning_rate": 1.677833383153542e-06,
      "logits/chosen": 6.4255805015563965,
      "logits/rejected": 5.784458160400391,
      "logps/chosen": -4.204034328460693,
      "logps/rejected": -6.822949409484863,
      "loss": 0.3277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.306051731109619,
      "rewards/margins": 3.928372621536255,
      "rewards/rejected": -10.234424591064453,
      "step": 910
    },
    {
      "epoch": 1.1676190476190476,
      "grad_norm": 3.5059311389923096,
      "learning_rate": 1.6285698816954626e-06,
      "logits/chosen": 5.253472328186035,
      "logits/rejected": 5.226204872131348,
      "logps/chosen": -3.458686113357544,
      "logps/rejected": -5.801349639892578,
      "loss": 0.3212,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.188028812408447,
      "rewards/margins": 3.51399564743042,
      "rewards/rejected": -8.702024459838867,
      "step": 920
    },
    {
      "epoch": 1.1803174603174602,
      "grad_norm": 5.19541072845459,
      "learning_rate": 1.5796886182883053e-06,
      "logits/chosen": 5.724546909332275,
      "logits/rejected": 5.355966091156006,
      "logps/chosen": -4.142882347106934,
      "logps/rejected": -7.1851396560668945,
      "loss": 0.3472,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.214322566986084,
      "rewards/margins": 4.563385963439941,
      "rewards/rejected": -10.777708053588867,
      "step": 930
    },
    {
      "epoch": 1.193015873015873,
      "grad_norm": 3.7898037433624268,
      "learning_rate": 1.5312110338697427e-06,
      "logits/chosen": 5.5762104988098145,
      "logits/rejected": 5.337274551391602,
      "logps/chosen": -3.7495741844177246,
      "logps/rejected": -6.201944351196289,
      "loss": 0.3193,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.624361515045166,
      "rewards/margins": 3.6785552501678467,
      "rewards/rejected": -9.30291748046875,
      "step": 940
    },
    {
      "epoch": 1.2057142857142857,
      "grad_norm": 8.723204612731934,
      "learning_rate": 1.4831583923105e-06,
      "logits/chosen": 6.4793524742126465,
      "logits/rejected": 6.682563781738281,
      "logps/chosen": -3.6695919036865234,
      "logps/rejected": -6.200020790100098,
      "loss": 0.3267,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.504387855529785,
      "rewards/margins": 3.795642375946045,
      "rewards/rejected": -9.300030708312988,
      "step": 950
    },
    {
      "epoch": 1.2057142857142857,
      "eval_logits/chosen": 5.892926216125488,
      "eval_logits/rejected": 5.720220565795898,
      "eval_logps/chosen": -3.7383785247802734,
      "eval_logps/rejected": -6.7838873863220215,
      "eval_loss": 0.27200350165367126,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -5.60756778717041,
      "eval_rewards/margins": 4.568263053894043,
      "eval_rewards/rejected": -10.175830841064453,
      "eval_runtime": 18.3385,
      "eval_samples_per_second": 27.81,
      "eval_steps_per_second": 3.49,
      "step": 950
    },
    {
      "epoch": 1.2184126984126984,
      "grad_norm": 4.321587085723877,
      "learning_rate": 1.4355517710873184e-06,
      "logits/chosen": 5.534070014953613,
      "logits/rejected": 5.096575736999512,
      "logps/chosen": -4.546253204345703,
      "logps/rejected": -7.369414329528809,
      "loss": 0.3554,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.8193793296813965,
      "rewards/margins": 4.2347412109375,
      "rewards/rejected": -11.054121017456055,
      "step": 960
    },
    {
      "epoch": 1.231111111111111,
      "grad_norm": 2.3186330795288086,
      "learning_rate": 1.388412052037682e-06,
      "logits/chosen": 6.364870548248291,
      "logits/rejected": 6.031035423278809,
      "logps/chosen": -3.9153428077697754,
      "logps/rejected": -6.4653639793396,
      "loss": 0.2867,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.873013973236084,
      "rewards/margins": 3.825031280517578,
      "rewards/rejected": -9.69804573059082,
      "step": 970
    },
    {
      "epoch": 1.243809523809524,
      "grad_norm": 6.711607456207275,
      "learning_rate": 1.3417599122003464e-06,
      "logits/chosen": 6.216970443725586,
      "logits/rejected": 5.668035507202148,
      "logps/chosen": -4.147327899932861,
      "logps/rejected": -7.050875186920166,
      "loss": 0.3064,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.220992088317871,
      "rewards/margins": 4.355320930480957,
      "rewards/rejected": -10.576312065124512,
      "step": 980
    },
    {
      "epoch": 1.2565079365079366,
      "grad_norm": 4.946932792663574,
      "learning_rate": 1.2956158147457116e-06,
      "logits/chosen": 5.909533500671387,
      "logits/rejected": 5.650919437408447,
      "logps/chosen": -4.229696273803711,
      "logps/rejected": -7.213942050933838,
      "loss": 0.3079,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.344544410705566,
      "rewards/margins": 4.476368427276611,
      "rewards/rejected": -10.820913314819336,
      "step": 990
    },
    {
      "epoch": 1.2692063492063492,
      "grad_norm": 3.831132411956787,
      "learning_rate": 1.2500000000000007e-06,
      "logits/chosen": 4.852212429046631,
      "logits/rejected": 4.722367763519287,
      "logps/chosen": -4.436448574066162,
      "logps/rejected": -7.6168317794799805,
      "loss": 0.3158,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.6546735763549805,
      "rewards/margins": 4.770575046539307,
      "rewards/rejected": -11.425248146057129,
      "step": 1000
    },
    {
      "epoch": 1.2692063492063492,
      "eval_logits/chosen": 5.721224784851074,
      "eval_logits/rejected": 5.573430061340332,
      "eval_logps/chosen": -3.8567862510681152,
      "eval_logps/rejected": -6.927263259887695,
      "eval_loss": 0.26795223355293274,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -5.785179615020752,
      "eval_rewards/margins": 4.605715751647949,
      "eval_rewards/rejected": -10.390894889831543,
      "eval_runtime": 18.3572,
      "eval_samples_per_second": 27.782,
      "eval_steps_per_second": 3.486,
      "step": 1000
    },
    {
      "epoch": 1.2819047619047619,
      "grad_norm": 4.340210914611816,
      "learning_rate": 1.204932476567175e-06,
      "logits/chosen": 5.455284595489502,
      "logits/rejected": 5.007905006408691,
      "logps/chosen": -4.618704795837402,
      "logps/rejected": -7.544007778167725,
      "loss": 0.3142,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.928057670593262,
      "rewards/margins": 4.387955188751221,
      "rewards/rejected": -11.316012382507324,
      "step": 1010
    },
    {
      "epoch": 1.2946031746031745,
      "grad_norm": 3.371431350708008,
      "learning_rate": 1.160433012552508e-06,
      "logits/chosen": 5.39217472076416,
      "logits/rejected": 5.3231329917907715,
      "logps/chosen": -3.9590423107147217,
      "logps/rejected": -7.2238945960998535,
      "loss": 0.3226,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.938564300537109,
      "rewards/margins": 4.897278785705566,
      "rewards/rejected": -10.835843086242676,
      "step": 1020
    },
    {
      "epoch": 1.3073015873015872,
      "grad_norm": 3.215897560119629,
      "learning_rate": 1.11652112689164e-06,
      "logits/chosen": 5.496011734008789,
      "logits/rejected": 5.371179580688477,
      "logps/chosen": -4.006840705871582,
      "logps/rejected": -6.894573211669922,
      "loss": 0.2998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.0102620124816895,
      "rewards/margins": 4.331597805023193,
      "rewards/rejected": -10.341859817504883,
      "step": 1030
    },
    {
      "epoch": 1.32,
      "grad_norm": 2.8000078201293945,
      "learning_rate": 1.073216080788921e-06,
      "logits/chosen": 6.246156215667725,
      "logits/rejected": 5.520369052886963,
      "logps/chosen": -4.688235282897949,
      "logps/rejected": -8.393880844116211,
      "loss": 0.2691,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.032352447509766,
      "rewards/margins": 5.558468341827393,
      "rewards/rejected": -12.590821266174316,
      "step": 1040
    },
    {
      "epoch": 1.3326984126984127,
      "grad_norm": 3.0592737197875977,
      "learning_rate": 1.0305368692688175e-06,
      "logits/chosen": 5.306058406829834,
      "logits/rejected": 5.3769707679748535,
      "logps/chosen": -4.188882350921631,
      "logps/rejected": -6.395564079284668,
      "loss": 0.3026,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.283323764801025,
      "rewards/margins": 3.3100223541259766,
      "rewards/rejected": -9.593345642089844,
      "step": 1050
    },
    {
      "epoch": 1.3326984126984127,
      "eval_logits/chosen": 5.6976423263549805,
      "eval_logits/rejected": 5.525759696960449,
      "eval_logps/chosen": -3.8439218997955322,
      "eval_logps/rejected": -6.9627227783203125,
      "eval_loss": 0.26340675354003906,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -5.76588249206543,
      "eval_rewards/margins": 4.678201675415039,
      "eval_rewards/rejected": -10.444085121154785,
      "eval_runtime": 18.3435,
      "eval_samples_per_second": 27.803,
      "eval_steps_per_second": 3.489,
      "step": 1050
    },
    {
      "epoch": 1.3453968253968254,
      "grad_norm": 3.524646759033203,
      "learning_rate": 9.88502212844063e-07,
      "logits/chosen": 5.86399507522583,
      "logits/rejected": 5.444242000579834,
      "logps/chosen": -4.269190788269043,
      "logps/rejected": -7.391558647155762,
      "loss": 0.2872,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.403787136077881,
      "rewards/margins": 4.68355131149292,
      "rewards/rejected": -11.087339401245117,
      "step": 1060
    },
    {
      "epoch": 1.358095238095238,
      "grad_norm": 3.6155037879943848,
      "learning_rate": 9.471305493042243e-07,
      "logits/chosen": 6.007330894470215,
      "logits/rejected": 5.697009563446045,
      "logps/chosen": -4.269715785980225,
      "logps/rejected": -7.014040470123291,
      "loss": 0.2794,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.404573917388916,
      "rewards/margins": 4.116486072540283,
      "rewards/rejected": -10.5210599899292,
      "step": 1070
    },
    {
      "epoch": 1.370793650793651,
      "grad_norm": 4.943655014038086,
      "learning_rate": 9.064400256282757e-07,
      "logits/chosen": 5.4075846672058105,
      "logits/rejected": 5.163808345794678,
      "logps/chosen": -3.780120849609375,
      "logps/rejected": -6.776060581207275,
      "loss": 0.3154,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.6701812744140625,
      "rewards/margins": 4.4939093589782715,
      "rewards/rejected": -10.164091110229492,
      "step": 1080
    },
    {
      "epoch": 1.3834920634920636,
      "grad_norm": 3.8432955741882324,
      "learning_rate": 8.664484900247363e-07,
      "logits/chosen": 6.4478631019592285,
      "logits/rejected": 5.634939193725586,
      "logps/chosen": -4.478579521179199,
      "logps/rejected": -7.522899627685547,
      "loss": 0.3541,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.717869758605957,
      "rewards/margins": 4.566478729248047,
      "rewards/rejected": -11.28434944152832,
      "step": 1090
    },
    {
      "epoch": 1.3961904761904762,
      "grad_norm": 5.117826461791992,
      "learning_rate": 8.271734841028553e-07,
      "logits/chosen": 5.143468379974365,
      "logits/rejected": 5.166830539703369,
      "logps/chosen": -3.964787721633911,
      "logps/rejected": -7.12085485458374,
      "loss": 0.3048,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.9471821784973145,
      "rewards/margins": 4.734100818634033,
      "rewards/rejected": -10.681283950805664,
      "step": 1100
    },
    {
      "epoch": 1.3961904761904762,
      "eval_logits/chosen": 5.741722583770752,
      "eval_logits/rejected": 5.563231468200684,
      "eval_logps/chosen": -3.90653395652771,
      "eval_logps/rejected": -7.086528778076172,
      "eval_loss": 0.2575596570968628,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -5.859801292419434,
      "eval_rewards/margins": 4.769991874694824,
      "eval_rewards/rejected": -10.629792213439941,
      "eval_runtime": 18.3447,
      "eval_samples_per_second": 27.801,
      "eval_steps_per_second": 3.489,
      "step": 1100
    }
  ],
  "logging_steps": 10,
  "max_steps": 1500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 2.653959905374896e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}