checkpoint-650/trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.8253968253968254,
  "eval_steps": 50,
  "global_step": 650,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.012698412698412698,
      "grad_norm": 0.04716634005308151,
      "learning_rate": 4.999451708687114e-06,
      "logits/chosen": 14.883691787719727,
      "logits/rejected": 15.016583442687988,
      "logps/chosen": -0.29512909054756165,
      "logps/rejected": -0.30033987760543823,
      "loss": 1.0007,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.4426936209201813,
      "rewards/margins": 0.007816222496330738,
      "rewards/rejected": -0.45050984621047974,
      "step": 10
    },
    {
      "epoch": 0.025396825396825397,
      "grad_norm": 0.04392225295305252,
      "learning_rate": 4.997807075247147e-06,
      "logits/chosen": 14.705224990844727,
      "logits/rejected": 14.737253189086914,
      "logps/chosen": -0.3201700747013092,
      "logps/rejected": -0.280050128698349,
      "loss": 0.9984,
      "rewards/accuracies": 0.26249998807907104,
      "rewards/chosen": -0.4802550673484802,
      "rewards/margins": -0.060179851949214935,
      "rewards/rejected": -0.4200752377510071,
      "step": 20
    },
    {
      "epoch": 0.0380952380952381,
      "grad_norm": 0.04989476501941681,
      "learning_rate": 4.9950668210706795e-06,
      "logits/chosen": 15.147977828979492,
      "logits/rejected": 15.080279350280762,
      "logps/chosen": -0.3157380223274231,
      "logps/rejected": -0.2997627556324005,
      "loss": 0.9991,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": -0.47360706329345703,
      "rewards/margins": -0.023962898179888725,
      "rewards/rejected": -0.44964417815208435,
      "step": 30
    },
    {
      "epoch": 0.050793650793650794,
      "grad_norm": 0.054729390889406204,
      "learning_rate": 4.9912321481237616e-06,
      "logits/chosen": 15.301165580749512,
      "logits/rejected": 15.276555061340332,
      "logps/chosen": -0.30470195412635803,
      "logps/rejected": -0.29767152667045593,
      "loss": 0.9849,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.45705294609069824,
      "rewards/margins": -0.010545584373176098,
      "rewards/rejected": -0.44650736451148987,
      "step": 40
    },
    {
      "epoch": 0.06349206349206349,
      "grad_norm": 0.06035450100898743,
      "learning_rate": 4.986304738420684e-06,
      "logits/chosen": 14.600168228149414,
      "logits/rejected": 14.7944917678833,
      "logps/chosen": -0.32278841733932495,
      "logps/rejected": -0.3014402687549591,
      "loss": 0.9991,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.48418259620666504,
      "rewards/margins": -0.03202226758003235,
      "rewards/rejected": -0.4521603584289551,
      "step": 50
    },
    {
      "epoch": 0.06349206349206349,
      "eval_logits/chosen": 15.261337280273438,
      "eval_logits/rejected": 15.51547908782959,
      "eval_logps/chosen": -0.3022651970386505,
      "eval_logps/rejected": -0.3061661124229431,
      "eval_loss": 0.9846106171607971,
      "eval_rewards/accuracies": 0.40625,
      "eval_rewards/chosen": -0.4533977508544922,
      "eval_rewards/margins": 0.005851435009390116,
      "eval_rewards/rejected": -0.45924919843673706,
      "eval_runtime": 18.4033,
      "eval_samples_per_second": 27.712,
      "eval_steps_per_second": 3.478,
      "step": 50
    },
    {
      "epoch": 0.0761904761904762,
      "grad_norm": 0.052623528987169266,
      "learning_rate": 4.980286753286196e-06,
      "logits/chosen": 15.2726469039917,
      "logits/rejected": 15.245884895324707,
      "logps/chosen": -0.31394433975219727,
      "logps/rejected": -0.2805258333683014,
      "loss": 1.0063,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": -0.4709165096282959,
      "rewards/margins": -0.05012776702642441,
      "rewards/rejected": -0.4207887053489685,
      "step": 60
    },
    {
      "epoch": 0.08888888888888889,
      "grad_norm": 0.3512271046638489,
      "learning_rate": 4.973180832407471e-06,
      "logits/chosen": 15.302217483520508,
      "logits/rejected": 15.387911796569824,
      "logps/chosen": -0.30377697944641113,
      "logps/rejected": -0.29276782274246216,
      "loss": 0.9888,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.4556654393672943,
      "rewards/margins": -0.016513748094439507,
      "rewards/rejected": -0.43915170431137085,
      "step": 70
    },
    {
      "epoch": 0.10158730158730159,
      "grad_norm": 0.056646134704351425,
      "learning_rate": 4.964990092676263e-06,
      "logits/chosen": 15.843823432922363,
      "logits/rejected": 15.895855903625488,
      "logps/chosen": -0.3091468811035156,
      "logps/rejected": -0.27621737122535706,
      "loss": 0.9926,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -0.46372032165527344,
      "rewards/margins": -0.04939427971839905,
      "rewards/rejected": -0.4143260419368744,
      "step": 80
    },
    {
      "epoch": 0.11428571428571428,
      "grad_norm": 0.062348950654268265,
      "learning_rate": 4.9557181268217225e-06,
      "logits/chosen": 15.522564888000488,
      "logits/rejected": 15.576242446899414,
      "logps/chosen": -0.3092329502105713,
      "logps/rejected": -0.27670228481292725,
      "loss": 0.9812,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": -0.4638494551181793,
      "rewards/margins": -0.048796020448207855,
      "rewards/rejected": -0.41505345702171326,
      "step": 90
    },
    {
      "epoch": 0.12698412698412698,
      "grad_norm": 0.06779928505420685,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": 15.999185562133789,
      "logits/rejected": 15.802484512329102,
      "logps/chosen": -0.3173079490661621,
      "logps/rejected": -0.283602774143219,
      "loss": 0.981,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": -0.4759618639945984,
      "rewards/margins": -0.050557754933834076,
      "rewards/rejected": -0.4254041314125061,
      "step": 100
    },
    {
      "epoch": 0.12698412698412698,
      "eval_logits/chosen": 15.745854377746582,
      "eval_logits/rejected": 16.0076961517334,
      "eval_logps/chosen": -0.29866182804107666,
      "eval_logps/rejected": -0.3121251165866852,
      "eval_loss": 0.9764130115509033,
      "eval_rewards/accuracies": 0.453125,
      "eval_rewards/chosen": -0.4479926824569702,
      "eval_rewards/margins": 0.02019493840634823,
      "eval_rewards/rejected": -0.4681876599788666,
      "eval_runtime": 18.3383,
      "eval_samples_per_second": 27.811,
      "eval_steps_per_second": 3.49,
      "step": 100
    },
    {
      "epoch": 0.13968253968253969,
      "grad_norm": 0.07346926629543304,
      "learning_rate": 4.933947257182901e-06,
      "logits/chosen": 15.723541259765625,
      "logits/rejected": 15.764880180358887,
      "logps/chosen": -0.3212783634662628,
      "logps/rejected": -0.3159862160682678,
      "loss": 0.9859,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.4819175601005554,
      "rewards/margins": -0.00793826300650835,
      "rewards/rejected": -0.47397929430007935,
      "step": 110
    },
    {
      "epoch": 0.1523809523809524,
      "grad_norm": 0.06748441606760025,
      "learning_rate": 4.921457902821578e-06,
      "logits/chosen": 15.817390441894531,
      "logits/rejected": 16.023067474365234,
      "logps/chosen": -0.29811763763427734,
      "logps/rejected": -0.30654552578926086,
      "loss": 0.9827,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.447176456451416,
      "rewards/margins": 0.01264181174337864,
      "rewards/rejected": -0.4598182737827301,
      "step": 120
    },
    {
      "epoch": 0.16507936507936508,
      "grad_norm": 0.08042758703231812,
      "learning_rate": 4.907906416994146e-06,
      "logits/chosen": 15.45033073425293,
      "logits/rejected": 15.865328788757324,
      "logps/chosen": -0.27259278297424316,
      "logps/rejected": -0.31338179111480713,
      "loss": 0.9743,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.40888920426368713,
      "rewards/margins": 0.06118353083729744,
      "rewards/rejected": -0.4700726866722107,
      "step": 130
    },
    {
      "epoch": 0.17777777777777778,
      "grad_norm": 0.15669430792331696,
      "learning_rate": 4.893298743830168e-06,
      "logits/chosen": 16.17765235900879,
      "logits/rejected": 15.951130867004395,
      "logps/chosen": -0.3015133738517761,
      "logps/rejected": -0.2970424294471741,
      "loss": 0.9824,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.4522700905799866,
      "rewards/margins": -0.006706444080919027,
      "rewards/rejected": -0.44556355476379395,
      "step": 140
    },
    {
      "epoch": 0.19047619047619047,
      "grad_norm": 0.07115967571735382,
      "learning_rate": 4.8776412907378845e-06,
      "logits/chosen": 15.552728652954102,
      "logits/rejected": 15.788042068481445,
      "logps/chosen": -0.28624147176742554,
      "logps/rejected": -0.3222900331020355,
      "loss": 0.9727,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4293622076511383,
      "rewards/margins": 0.05407290905714035,
      "rewards/rejected": -0.48343515396118164,
      "step": 150
    },
    {
      "epoch": 0.19047619047619047,
      "eval_logits/chosen": 15.72143268585205,
      "eval_logits/rejected": 15.974818229675293,
      "eval_logps/chosen": -0.29440543055534363,
      "eval_logps/rejected": -0.33279579877853394,
      "eval_loss": 0.9540849328041077,
      "eval_rewards/accuracies": 0.546875,
      "eval_rewards/chosen": -0.44160816073417664,
      "eval_rewards/margins": 0.05758553743362427,
      "eval_rewards/rejected": -0.4991936683654785,
      "eval_runtime": 18.335,
      "eval_samples_per_second": 27.816,
      "eval_steps_per_second": 3.491,
      "step": 150
    },
    {
      "epoch": 0.20317460317460317,
      "grad_norm": 0.1144075095653534,
      "learning_rate": 4.860940925593703e-06,
      "logits/chosen": 15.63622760772705,
      "logits/rejected": 15.758941650390625,
      "logps/chosen": -0.310846745967865,
      "logps/rejected": -0.35185256600379944,
      "loss": 0.9517,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.4662701487541199,
      "rewards/margins": 0.061508744955062866,
      "rewards/rejected": -0.5277789235115051,
      "step": 160
    },
    {
      "epoch": 0.21587301587301588,
      "grad_norm": 0.1173442155122757,
      "learning_rate": 4.84320497372973e-06,
      "logits/chosen": 15.8660306930542,
      "logits/rejected": 15.83642292022705,
      "logps/chosen": -0.279310017824173,
      "logps/rejected": -0.3301998972892761,
      "loss": 0.9439,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.41896501183509827,
      "rewards/margins": 0.07633484899997711,
      "rewards/rejected": -0.49529990553855896,
      "step": 170
    },
    {
      "epoch": 0.22857142857142856,
      "grad_norm": 0.204214945435524,
      "learning_rate": 4.824441214720629e-06,
      "logits/chosen": 15.557527542114258,
      "logits/rejected": 15.321504592895508,
      "logps/chosen": -0.3029389977455139,
      "logps/rejected": -0.3163699507713318,
      "loss": 0.9369,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.4544084668159485,
      "rewards/margins": 0.020146463066339493,
      "rewards/rejected": -0.47455495595932007,
      "step": 180
    },
    {
      "epoch": 0.24126984126984127,
      "grad_norm": 0.14012843370437622,
      "learning_rate": 4.804657878971252e-06,
      "logits/chosen": 15.567469596862793,
      "logits/rejected": 15.717155456542969,
      "logps/chosen": -0.3400501310825348,
      "logps/rejected": -0.3475819230079651,
      "loss": 0.9443,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.5100752115249634,
      "rewards/margins": 0.011297682300209999,
      "rewards/rejected": -0.5213728547096252,
      "step": 190
    },
    {
      "epoch": 0.25396825396825395,
      "grad_norm": 0.12908300757408142,
      "learning_rate": 4.783863644106502e-06,
      "logits/chosen": 14.985162734985352,
      "logits/rejected": 15.223039627075195,
      "logps/chosen": -0.2876646816730499,
      "logps/rejected": -0.3777172267436981,
      "loss": 0.9034,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.4314970076084137,
      "rewards/margins": 0.13507887721061707,
      "rewards/rejected": -0.5665758848190308,
      "step": 200
    },
    {
      "epoch": 0.25396825396825395,
      "eval_logits/chosen": 15.242753028869629,
      "eval_logits/rejected": 15.523069381713867,
      "eval_logps/chosen": -0.2961229085922241,
      "eval_logps/rejected": -0.3966684341430664,
      "eval_loss": 0.9075753092765808,
      "eval_rewards/accuracies": 0.59375,
      "eval_rewards/chosen": -0.4441843330860138,
      "eval_rewards/margins": 0.1508183479309082,
      "eval_rewards/rejected": -0.5950026512145996,
      "eval_runtime": 18.341,
      "eval_samples_per_second": 27.807,
      "eval_steps_per_second": 3.489,
      "step": 200
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 0.13152211904525757,
      "learning_rate": 4.762067631165049e-06,
      "logits/chosen": 15.283185958862305,
      "logits/rejected": 15.545951843261719,
      "logps/chosen": -0.30045825242996216,
      "logps/rejected": -0.4519944190979004,
      "loss": 0.9085,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.45068734884262085,
      "rewards/margins": 0.22730426490306854,
      "rewards/rejected": -0.6779916286468506,
      "step": 210
    },
    {
      "epoch": 0.27936507936507937,
      "grad_norm": 0.14594660699367523,
      "learning_rate": 4.7392794005985324e-06,
      "logits/chosen": 15.53101921081543,
      "logits/rejected": 15.322232246398926,
      "logps/chosen": -0.3336530327796936,
      "logps/rejected": -0.36337023973464966,
      "loss": 0.9188,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.5004795789718628,
      "rewards/margins": 0.04457578808069229,
      "rewards/rejected": -0.5450553894042969,
      "step": 220
    },
    {
      "epoch": 0.2920634920634921,
      "grad_norm": 0.14854153990745544,
      "learning_rate": 4.715508948078037e-06,
      "logits/chosen": 14.157003402709961,
      "logits/rejected": 14.34089183807373,
      "logps/chosen": -0.28324443101882935,
      "logps/rejected": -0.41318759322166443,
      "loss": 0.9015,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.4248666763305664,
      "rewards/margins": 0.19491472840309143,
      "rewards/rejected": -0.6197813749313354,
      "step": 230
    },
    {
      "epoch": 0.3047619047619048,
      "grad_norm": 0.19532087445259094,
      "learning_rate": 4.690766700109659e-06,
      "logits/chosen": 14.729398727416992,
      "logits/rejected": 14.97203254699707,
      "logps/chosen": -0.3013172149658203,
      "logps/rejected": -0.41750937700271606,
      "loss": 0.8814,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.45197582244873047,
      "rewards/margins": 0.17428824305534363,
      "rewards/rejected": -0.6262640357017517,
      "step": 240
    },
    {
      "epoch": 0.31746031746031744,
      "grad_norm": 0.5795227885246277,
      "learning_rate": 4.665063509461098e-06,
      "logits/chosen": 14.328412055969238,
      "logits/rejected": 14.645428657531738,
      "logps/chosen": -0.3055194914340973,
      "logps/rejected": -0.509621262550354,
      "loss": 0.8478,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.45827922224998474,
      "rewards/margins": 0.30615273118019104,
      "rewards/rejected": -0.764431893825531,
      "step": 250
    },
    {
      "epoch": 0.31746031746031744,
      "eval_logits/chosen": 14.310781478881836,
      "eval_logits/rejected": 14.5652494430542,
      "eval_logps/chosen": -0.3155084252357483,
      "eval_logps/rejected": -0.8301784992218018,
      "eval_loss": 0.7700436115264893,
      "eval_rewards/accuracies": 0.640625,
      "eval_rewards/chosen": -0.47326260805130005,
      "eval_rewards/margins": 0.7720052003860474,
      "eval_rewards/rejected": -1.2452678680419922,
      "eval_runtime": 18.3413,
      "eval_samples_per_second": 27.806,
      "eval_steps_per_second": 3.489,
      "step": 250
    },
    {
      "epoch": 0.33015873015873015,
      "grad_norm": 0.24876689910888672,
      "learning_rate": 4.638410650401267e-06,
      "logits/chosen": 14.115270614624023,
      "logits/rejected": 14.167577743530273,
      "logps/chosen": -0.32328444719314575,
      "logps/rejected": -0.8564422726631165,
      "loss": 0.795,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.484926700592041,
      "rewards/margins": 0.7997367978096008,
      "rewards/rejected": -1.284663438796997,
      "step": 260
    },
    {
      "epoch": 0.34285714285714286,
      "grad_norm": 0.5779634118080139,
      "learning_rate": 4.610819813755038e-06,
      "logits/chosen": 14.019407272338867,
      "logits/rejected": 13.600751876831055,
      "logps/chosen": -0.35225847363471985,
      "logps/rejected": -1.1924464702606201,
      "loss": 0.7644,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5283876657485962,
      "rewards/margins": 1.2602821588516235,
      "rewards/rejected": -1.7886695861816406,
      "step": 270
    },
    {
      "epoch": 0.35555555555555557,
      "grad_norm": 0.2085695117712021,
      "learning_rate": 4.582303101775249e-06,
      "logits/chosen": 13.448820114135742,
      "logits/rejected": 13.464624404907227,
      "logps/chosen": -0.3442023992538452,
      "logps/rejected": -1.4783053398132324,
      "loss": 0.7671,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.516303539276123,
      "rewards/margins": 1.7011544704437256,
      "rewards/rejected": -2.2174580097198486,
      "step": 280
    },
    {
      "epoch": 0.3682539682539683,
      "grad_norm": 0.2825860381126404,
      "learning_rate": 4.55287302283426e-06,
      "logits/chosen": 13.038851737976074,
      "logits/rejected": 13.013893127441406,
      "logps/chosen": -0.3524690568447113,
      "logps/rejected": -1.2354861497879028,
      "loss": 0.7922,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5287035703659058,
      "rewards/margins": 1.3245255947113037,
      "rewards/rejected": -1.8532291650772095,
      "step": 290
    },
    {
      "epoch": 0.38095238095238093,
      "grad_norm": 0.30771639943122864,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": 13.67170524597168,
      "logits/rejected": 13.185449600219727,
      "logps/chosen": -0.3915162682533264,
      "logps/rejected": -1.3473880290985107,
      "loss": 0.7506,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.587274432182312,
      "rewards/margins": 1.433807611465454,
      "rewards/rejected": -2.0210821628570557,
      "step": 300
    },
    {
      "epoch": 0.38095238095238093,
      "eval_logits/chosen": 13.198552131652832,
      "eval_logits/rejected": 13.458879470825195,
      "eval_logps/chosen": -0.37132135033607483,
      "eval_logps/rejected": -1.4595866203308105,
      "eval_loss": 0.7391816973686218,
      "eval_rewards/accuracies": 0.640625,
      "eval_rewards/chosen": -0.5569820404052734,
      "eval_rewards/margins": 1.6323981285095215,
      "eval_rewards/rejected": -2.189380168914795,
      "eval_runtime": 18.3117,
      "eval_samples_per_second": 27.851,
      "eval_steps_per_second": 3.495,
      "step": 300
    },
    {
      "epoch": 0.39365079365079364,
      "grad_norm": 0.3021286129951477,
      "learning_rate": 4.491324795060491e-06,
      "logits/chosen": 13.347516059875488,
      "logits/rejected": 13.285726547241211,
      "logps/chosen": -0.4108191132545471,
      "logps/rejected": -1.436702013015747,
      "loss": 0.7134,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.6162286400794983,
      "rewards/margins": 1.5388243198394775,
      "rewards/rejected": -2.15505313873291,
      "step": 310
    },
    {
      "epoch": 0.40634920634920635,
      "grad_norm": 0.40123897790908813,
      "learning_rate": 4.4592336433146e-06,
      "logits/chosen": 12.949071884155273,
      "logits/rejected": 12.86626148223877,
      "logps/chosen": -0.42898029088974,
      "logps/rejected": -1.1741807460784912,
      "loss": 0.74,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.6434704065322876,
      "rewards/margins": 1.1178009510040283,
      "rewards/rejected": -1.7612712383270264,
      "step": 320
    },
    {
      "epoch": 0.41904761904761906,
      "grad_norm": 0.4613189697265625,
      "learning_rate": 4.426283106939474e-06,
      "logits/chosen": 12.796838760375977,
      "logits/rejected": 12.524165153503418,
      "logps/chosen": -0.48975634574890137,
      "logps/rejected": -1.2766364812850952,
      "loss": 0.7408,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7346345782279968,
      "rewards/margins": 1.180320143699646,
      "rewards/rejected": -1.9149547815322876,
      "step": 330
    },
    {
      "epoch": 0.43174603174603177,
      "grad_norm": 0.49141305685043335,
      "learning_rate": 4.3924876391293915e-06,
      "logits/chosen": 12.58080005645752,
      "logits/rejected": 12.5891695022583,
      "logps/chosen": -0.5303409099578857,
      "logps/rejected": -1.6262977123260498,
      "loss": 0.7501,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.7955113649368286,
      "rewards/margins": 1.643935203552246,
      "rewards/rejected": -2.4394466876983643,
      "step": 340
    },
    {
      "epoch": 0.4444444444444444,
      "grad_norm": 0.7533912062644958,
      "learning_rate": 4.357862063693486e-06,
      "logits/chosen": 12.15321159362793,
      "logits/rejected": 12.38386344909668,
      "logps/chosen": -0.5782582759857178,
      "logps/rejected": -1.580635905265808,
      "loss": 0.7405,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8673874139785767,
      "rewards/margins": 1.5035666227340698,
      "rewards/rejected": -2.3709540367126465,
      "step": 350
    },
    {
      "epoch": 0.4444444444444444,
      "eval_logits/chosen": 12.111034393310547,
      "eval_logits/rejected": 12.33246898651123,
      "eval_logps/chosen": -0.5941734910011292,
      "eval_logps/rejected": -1.6774102449417114,
      "eval_loss": 0.6953701972961426,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": -0.8912601470947266,
      "eval_rewards/margins": 1.6248550415039062,
      "eval_rewards/rejected": -2.516115188598633,
      "eval_runtime": 18.3366,
      "eval_samples_per_second": 27.813,
      "eval_steps_per_second": 3.49,
      "step": 350
    },
    {
      "epoch": 0.45714285714285713,
      "grad_norm": 1.168564796447754,
      "learning_rate": 4.322421568553529e-06,
      "logits/chosen": 11.706710815429688,
      "logits/rejected": 11.451011657714844,
      "logps/chosen": -0.758882462978363,
      "logps/rejected": -1.842167615890503,
      "loss": 0.702,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.1383236646652222,
      "rewards/margins": 1.6249277591705322,
      "rewards/rejected": -2.763251304626465,
      "step": 360
    },
    {
      "epoch": 0.46984126984126984,
      "grad_norm": 1.8962547779083252,
      "learning_rate": 4.286181699082008e-06,
      "logits/chosen": 11.41061019897461,
      "logits/rejected": 11.534398078918457,
      "logps/chosen": -1.1785552501678467,
      "logps/rejected": -2.235652446746826,
      "loss": 0.6781,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7678327560424805,
      "rewards/margins": 1.5856454372406006,
      "rewards/rejected": -3.3534786701202393,
      "step": 370
    },
    {
      "epoch": 0.48253968253968255,
      "grad_norm": 2.9152028560638428,
      "learning_rate": 4.249158351283414e-06,
      "logits/chosen": 11.582561492919922,
      "logits/rejected": 11.439531326293945,
      "logps/chosen": -1.9169971942901611,
      "logps/rejected": -2.788179397583008,
      "loss": 0.6033,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.875495672225952,
      "rewards/margins": 1.306773066520691,
      "rewards/rejected": -4.182269096374512,
      "step": 380
    },
    {
      "epoch": 0.49523809523809526,
      "grad_norm": 1.5070641040802002,
      "learning_rate": 4.211367764821722e-06,
      "logits/chosen": 11.340807914733887,
      "logits/rejected": 10.927321434020996,
      "logps/chosen": -2.1075730323791504,
      "logps/rejected": -3.4216556549072266,
      "loss": 0.5698,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.1613595485687256,
      "rewards/margins": 1.9711239337921143,
      "rewards/rejected": -5.13248348236084,
      "step": 390
    },
    {
      "epoch": 0.5079365079365079,
      "grad_norm": 2.6913516521453857,
      "learning_rate": 4.172826515897146e-06,
      "logits/chosen": 10.273612976074219,
      "logits/rejected": 10.116204261779785,
      "logps/chosen": -3.0047056674957275,
      "logps/rejected": -4.053646564483643,
      "loss": 0.5869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.507058143615723,
      "rewards/margins": 1.5734113454818726,
      "rewards/rejected": -6.080469608306885,
      "step": 400
    },
    {
      "epoch": 0.5079365079365079,
      "eval_logits/chosen": 9.58273696899414,
      "eval_logits/rejected": 9.765192031860352,
      "eval_logps/chosen": -2.4895575046539307,
      "eval_logps/rejected": -3.8314545154571533,
      "eval_loss": 0.5232856273651123,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": -3.7343361377716064,
      "eval_rewards/margins": 2.012845754623413,
      "eval_rewards/rejected": -5.7471818923950195,
      "eval_runtime": 18.3437,
      "eval_samples_per_second": 27.802,
      "eval_steps_per_second": 3.489,
      "step": 400
    },
    {
      "epoch": 0.5206349206349207,
      "grad_norm": 3.7197885513305664,
      "learning_rate": 4.133551509975264e-06,
      "logits/chosen": 9.081324577331543,
      "logits/rejected": 9.309822082519531,
      "logps/chosen": -2.2554564476013184,
      "logps/rejected": -3.2624363899230957,
      "loss": 0.5774,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.3831849098205566,
      "rewards/margins": 1.5104694366455078,
      "rewards/rejected": -4.893653869628906,
      "step": 410
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 2.757873773574829,
      "learning_rate": 4.093559974371725e-06,
      "logits/chosen": 8.145927429199219,
      "logits/rejected": 8.402036666870117,
      "logps/chosen": -3.1110777854919434,
      "logps/rejected": -4.202320098876953,
      "loss": 0.5084,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -4.666616916656494,
      "rewards/margins": 1.636863350868225,
      "rewards/rejected": -6.30348014831543,
      "step": 420
    },
    {
      "epoch": 0.546031746031746,
      "grad_norm": 3.3764102458953857,
      "learning_rate": 4.052869450695776e-06,
      "logits/chosen": 8.132574081420898,
      "logits/rejected": 8.386279106140137,
      "logps/chosen": -3.0871124267578125,
      "logps/rejected": -4.399088382720947,
      "loss": 0.5233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.630668640136719,
      "rewards/margins": 1.967963457107544,
      "rewards/rejected": -6.598631858825684,
      "step": 430
    },
    {
      "epoch": 0.5587301587301587,
      "grad_norm": 1.8188632726669312,
      "learning_rate": 4.011497787155938e-06,
      "logits/chosen": 8.104793548583984,
      "logits/rejected": 8.25890064239502,
      "logps/chosen": -3.598665237426758,
      "logps/rejected": -5.2809929847717285,
      "loss": 0.5272,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.397997856140137,
      "rewards/margins": 2.5234923362731934,
      "rewards/rejected": -7.921489715576172,
      "step": 440
    },
    {
      "epoch": 0.5714285714285714,
      "grad_norm": 2.528883934020996,
      "learning_rate": 3.969463130731183e-06,
      "logits/chosen": 8.675338745117188,
      "logits/rejected": 8.550267219543457,
      "logps/chosen": -3.582223892211914,
      "logps/rejected": -5.077345848083496,
      "loss": 0.5717,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -5.373335838317871,
      "rewards/margins": 2.2426836490631104,
      "rewards/rejected": -7.616019248962402,
      "step": 450
    },
    {
      "epoch": 0.5714285714285714,
      "eval_logits/chosen": 8.125189781188965,
      "eval_logits/rejected": 8.239119529724121,
      "eval_logps/chosen": -3.107839584350586,
      "eval_logps/rejected": -4.726868629455566,
      "eval_loss": 0.4568406939506531,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": -4.661759376525879,
      "eval_rewards/margins": 2.4285435676574707,
      "eval_rewards/rejected": -7.090303421020508,
      "eval_runtime": 18.3222,
      "eval_samples_per_second": 27.835,
      "eval_steps_per_second": 3.493,
      "step": 450
    },
    {
      "epoch": 0.5841269841269842,
      "grad_norm": 2.5079057216644287,
      "learning_rate": 3.92678391921108e-06,
      "logits/chosen": 8.503251075744629,
      "logits/rejected": 8.457250595092773,
      "logps/chosen": -3.499116897583008,
      "logps/rejected": -5.12787389755249,
      "loss": 0.4803,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.2486748695373535,
      "rewards/margins": 2.4431352615356445,
      "rewards/rejected": -7.69180965423584,
      "step": 460
    },
    {
      "epoch": 0.5968253968253968,
      "grad_norm": 1.869667887687683,
      "learning_rate": 3.88347887310836e-06,
      "logits/chosen": 7.972044467926025,
      "logits/rejected": 7.825163841247559,
      "logps/chosen": -3.6900856494903564,
      "logps/rejected": -5.166090488433838,
      "loss": 0.4733,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.535128593444824,
      "rewards/margins": 2.2140071392059326,
      "rewards/rejected": -7.749135494232178,
      "step": 470
    },
    {
      "epoch": 0.6095238095238096,
      "grad_norm": 2.8777735233306885,
      "learning_rate": 3.839566987447492e-06,
      "logits/chosen": 8.073511123657227,
      "logits/rejected": 8.345193862915039,
      "logps/chosen": -3.7006676197052,
      "logps/rejected": -5.271113395690918,
      "loss": 0.5048,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.551001071929932,
      "rewards/margins": 2.3556694984436035,
      "rewards/rejected": -7.906670570373535,
      "step": 480
    },
    {
      "epoch": 0.6222222222222222,
      "grad_norm": 2.7042839527130127,
      "learning_rate": 3.795067523432826e-06,
      "logits/chosen": 8.602261543273926,
      "logits/rejected": 8.32430362701416,
      "logps/chosen": -3.9612879753112793,
      "logps/rejected": -6.123431205749512,
      "loss": 0.4728,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.94193172454834,
      "rewards/margins": 3.2432150840759277,
      "rewards/rejected": -9.185147285461426,
      "step": 490
    },
    {
      "epoch": 0.6349206349206349,
      "grad_norm": 2.1625821590423584,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": 7.857954978942871,
      "logits/rejected": 7.460604190826416,
      "logps/chosen": -3.581023693084717,
      "logps/rejected": -5.477323532104492,
      "loss": 0.4296,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.371535778045654,
      "rewards/margins": 2.844449520111084,
      "rewards/rejected": -8.215986251831055,
      "step": 500
    },
    {
      "epoch": 0.6349206349206349,
      "eval_logits/chosen": 7.520671844482422,
      "eval_logits/rejected": 7.586723327636719,
      "eval_logps/chosen": -3.1659929752349854,
      "eval_logps/rejected": -4.955198764801025,
      "eval_loss": 0.4266711175441742,
      "eval_rewards/accuracies": 0.890625,
      "eval_rewards/chosen": -4.748989105224609,
      "eval_rewards/margins": 2.683809518814087,
      "eval_rewards/rejected": -7.432799339294434,
      "eval_runtime": 18.3456,
      "eval_samples_per_second": 27.8,
      "eval_steps_per_second": 3.489,
      "step": 500
    },
    {
      "epoch": 0.6476190476190476,
      "grad_norm": 4.2689104080200195,
      "learning_rate": 3.7043841852542884e-06,
      "logits/chosen": 7.948687553405762,
      "logits/rejected": 7.656899929046631,
      "logps/chosen": -2.8974719047546387,
      "logps/rejected": -4.501969337463379,
      "loss": 0.4981,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.346207618713379,
      "rewards/margins": 2.4067459106445312,
      "rewards/rejected": -6.752954006195068,
      "step": 510
    },
    {
      "epoch": 0.6603174603174603,
      "grad_norm": 3.299290895462036,
      "learning_rate": 3.658240087799655e-06,
      "logits/chosen": 7.676476955413818,
      "logits/rejected": 7.332755088806152,
      "logps/chosen": -3.9289379119873047,
      "logps/rejected": -5.498073577880859,
      "loss": 0.4736,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -5.893406867980957,
      "rewards/margins": 2.353703498840332,
      "rewards/rejected": -8.247110366821289,
      "step": 520
    },
    {
      "epoch": 0.6730158730158731,
      "grad_norm": 3.1058828830718994,
      "learning_rate": 3.611587947962319e-06,
      "logits/chosen": 7.516615867614746,
      "logits/rejected": 7.344383239746094,
      "logps/chosen": -3.57861328125,
      "logps/rejected": -5.48795223236084,
      "loss": 0.502,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.367920398712158,
      "rewards/margins": 2.8640081882476807,
      "rewards/rejected": -8.231928825378418,
      "step": 530
    },
    {
      "epoch": 0.6857142857142857,
      "grad_norm": 3.874868869781494,
      "learning_rate": 3.564448228912682e-06,
      "logits/chosen": 7.619947910308838,
      "logits/rejected": 7.048402309417725,
      "logps/chosen": -4.164804935455322,
      "logps/rejected": -5.771735191345215,
      "loss": 0.44,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.247206687927246,
      "rewards/margins": 2.4103951454162598,
      "rewards/rejected": -8.657602310180664,
      "step": 540
    },
    {
      "epoch": 0.6984126984126984,
      "grad_norm": 1.7583000659942627,
      "learning_rate": 3.516841607689501e-06,
      "logits/chosen": 7.068112850189209,
      "logits/rejected": 7.042977333068848,
      "logps/chosen": -3.760105609893799,
      "logps/rejected": -5.3382248878479,
      "loss": 0.4751,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.640158653259277,
      "rewards/margins": 2.3671793937683105,
      "rewards/rejected": -8.00733757019043,
      "step": 550
    },
    {
      "epoch": 0.6984126984126984,
      "eval_logits/chosen": 7.213573455810547,
      "eval_logits/rejected": 7.25667667388916,
      "eval_logps/chosen": -3.8712658882141113,
      "eval_logps/rejected": -5.882909774780273,
      "eval_loss": 0.3922204077243805,
      "eval_rewards/accuracies": 0.90625,
      "eval_rewards/chosen": -5.806899070739746,
      "eval_rewards/margins": 3.017465591430664,
      "eval_rewards/rejected": -8.824363708496094,
      "eval_runtime": 18.3231,
      "eval_samples_per_second": 27.834,
      "eval_steps_per_second": 3.493,
      "step": 550
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 1.9696826934814453,
      "learning_rate": 3.4687889661302577e-06,
      "logits/chosen": 7.829404354095459,
      "logits/rejected": 7.19503927230835,
      "logps/chosen": -4.589760780334473,
      "logps/rejected": -6.637518405914307,
      "loss": 0.4127,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.884641170501709,
      "rewards/margins": 3.0716373920440674,
      "rewards/rejected": -9.956277847290039,
      "step": 560
    },
    {
      "epoch": 0.7238095238095238,
      "grad_norm": 4.987663269042969,
      "learning_rate": 3.4203113817116955e-06,
      "logits/chosen": 7.03420877456665,
      "logits/rejected": 7.229029655456543,
      "logps/chosen": -3.7655014991760254,
      "logps/rejected": -6.0161333084106445,
      "loss": 0.4436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.648251533508301,
      "rewards/margins": 3.375948667526245,
      "rewards/rejected": -9.024200439453125,
      "step": 570
    },
    {
      "epoch": 0.7365079365079366,
      "grad_norm": 2.468047618865967,
      "learning_rate": 3.3714301183045382e-06,
      "logits/chosen": 6.352839946746826,
      "logits/rejected": 6.344137668609619,
      "logps/chosen": -2.9854183197021484,
      "logps/rejected": -5.105284690856934,
      "loss": 0.411,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.478127479553223,
      "rewards/margins": 3.1797995567321777,
      "rewards/rejected": -7.657926082611084,
      "step": 580
    },
    {
      "epoch": 0.7492063492063492,
      "grad_norm": 2.321638822555542,
      "learning_rate": 3.3221666168464584e-06,
      "logits/chosen": 7.339761257171631,
      "logits/rejected": 7.080865383148193,
      "logps/chosen": -3.7470335960388184,
      "logps/rejected": -5.2733564376831055,
      "loss": 0.4297,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.62054967880249,
      "rewards/margins": 2.2894842624664307,
      "rewards/rejected": -7.9100341796875,
      "step": 590
    },
    {
      "epoch": 0.7619047619047619,
      "grad_norm": 6.850190162658691,
      "learning_rate": 3.272542485937369e-06,
      "logits/chosen": 7.278192043304443,
      "logits/rejected": 6.769750118255615,
      "logps/chosen": -3.9920971393585205,
      "logps/rejected": -5.948962211608887,
      "loss": 0.4537,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.988145351409912,
      "rewards/margins": 2.9352970123291016,
      "rewards/rejected": -8.923442840576172,
      "step": 600
    },
    {
      "epoch": 0.7619047619047619,
      "eval_logits/chosen": 6.745523929595947,
      "eval_logits/rejected": 6.732550144195557,
      "eval_logps/chosen": -3.667492628097534,
      "eval_logps/rejected": -5.859607696533203,
      "eval_loss": 0.368425577878952,
      "eval_rewards/accuracies": 0.921875,
      "eval_rewards/chosen": -5.501238822937012,
      "eval_rewards/margins": 3.288173198699951,
      "eval_rewards/rejected": -8.789411544799805,
      "eval_runtime": 18.3412,
      "eval_samples_per_second": 27.806,
      "eval_steps_per_second": 3.489,
      "step": 600
    },
    {
      "epoch": 0.7746031746031746,
      "grad_norm": 3.103388786315918,
      "learning_rate": 3.222579492361179e-06,
      "logits/chosen": 6.413697719573975,
      "logits/rejected": 6.046789646148682,
      "logps/chosen": -4.207709312438965,
      "logps/rejected": -6.105495452880859,
      "loss": 0.4315,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.311563968658447,
      "rewards/margins": 2.846680164337158,
      "rewards/rejected": -9.158244132995605,
      "step": 610
    },
    {
      "epoch": 0.7873015873015873,
      "grad_norm": 4.581955432891846,
      "learning_rate": 3.1722995515381644e-06,
      "logits/chosen": 6.37155818939209,
      "logits/rejected": 6.382074356079102,
      "logps/chosen": -4.053309917449951,
      "logps/rejected": -5.929131507873535,
      "loss": 0.4161,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.079965114593506,
      "rewards/margins": 2.813732624053955,
      "rewards/rejected": -8.893697738647461,
      "step": 620
    },
    {
      "epoch": 0.8,
      "grad_norm": 3.6864757537841797,
      "learning_rate": 3.121724717912138e-06,
      "logits/chosen": 5.701592445373535,
      "logits/rejected": 5.8575944900512695,
      "logps/chosen": -3.532477617263794,
      "logps/rejected": -5.849920749664307,
      "loss": 0.3621,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.2987165451049805,
      "rewards/margins": 3.4761645793914795,
      "rewards/rejected": -8.774881362915039,
      "step": 630
    },
    {
      "epoch": 0.8126984126984127,
      "grad_norm": 3.3179032802581787,
      "learning_rate": 3.0708771752766397e-06,
      "logits/chosen": 5.983328819274902,
      "logits/rejected": 5.853272914886475,
      "logps/chosen": -3.7127442359924316,
      "logps/rejected": -5.6633429527282715,
      "loss": 0.4325,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.569116115570068,
      "rewards/margins": 2.9258980751037598,
      "rewards/rejected": -8.495015144348145,
      "step": 640
    },
    {
      "epoch": 0.8253968253968254,
      "grad_norm": 3.3010666370391846,
      "learning_rate": 3.019779227044398e-06,
      "logits/chosen": 7.427163600921631,
      "logits/rejected": 7.144589900970459,
      "logps/chosen": -3.395155429840088,
      "logps/rejected": -5.485442638397217,
      "loss": 0.4376,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.0927324295043945,
      "rewards/margins": 3.1354317665100098,
      "rewards/rejected": -8.228163719177246,
      "step": 650
    },
    {
      "epoch": 0.8253968253968254,
      "eval_logits/chosen": 6.977372169494629,
      "eval_logits/rejected": 6.941837787628174,
      "eval_logps/chosen": -3.4915952682495117,
      "eval_logps/rejected": -5.880876541137695,
      "eval_loss": 0.3507956266403198,
      "eval_rewards/accuracies": 0.921875,
      "eval_rewards/chosen": -5.237393379211426,
      "eval_rewards/margins": 3.5839221477508545,
      "eval_rewards/rejected": -8.821314811706543,
      "eval_runtime": 18.3366,
      "eval_samples_per_second": 27.813,
      "eval_steps_per_second": 3.49,
      "step": 650
    }
  ],
  "logging_steps": 10,
  "max_steps": 1500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.5688698157061898e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}