checkpoint-1300/trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.050050454086781,
  "eval_steps": 50,
  "global_step": 1300,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.008072653884964682,
      "grad_norm": 0.04926518723368645,
      "learning_rate": 4.999451708687114e-06,
      "logits/chosen": 14.542106628417969,
      "logits/rejected": 14.864250183105469,
      "logps/chosen": -0.2809702754020691,
      "logps/rejected": -0.3013763725757599,
      "loss": 0.9221,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.421455442905426,
      "rewards/margins": 0.03060910664498806,
      "rewards/rejected": -0.452064573764801,
      "step": 10
    },
    {
      "epoch": 0.016145307769929364,
      "grad_norm": 0.056249432265758514,
      "learning_rate": 4.997807075247147e-06,
      "logits/chosen": 14.614748001098633,
      "logits/rejected": 15.259109497070312,
      "logps/chosen": -0.2828002870082855,
      "logps/rejected": -0.3477819561958313,
      "loss": 0.916,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.4242004454135895,
      "rewards/margins": 0.09747247397899628,
      "rewards/rejected": -0.5216729044914246,
      "step": 20
    },
    {
      "epoch": 0.024217961654894045,
      "grad_norm": 0.0664869099855423,
      "learning_rate": 4.9950668210706795e-06,
      "logits/chosen": 14.391204833984375,
      "logits/rejected": 14.82734203338623,
      "logps/chosen": -0.287629634141922,
      "logps/rejected": -0.3329126834869385,
      "loss": 0.914,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.4314444661140442,
      "rewards/margins": 0.06792456656694412,
      "rewards/rejected": -0.4993689954280853,
      "step": 30
    },
    {
      "epoch": 0.03229061553985873,
      "grad_norm": 0.055584829300642014,
      "learning_rate": 4.9912321481237616e-06,
      "logits/chosen": 13.93278980255127,
      "logits/rejected": 14.886846542358398,
      "logps/chosen": -0.28155946731567383,
      "logps/rejected": -0.3678051829338074,
      "loss": 0.9273,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.42233920097351074,
      "rewards/margins": 0.12936851382255554,
      "rewards/rejected": -0.5517078042030334,
      "step": 40
    },
    {
      "epoch": 0.04036326942482341,
      "grad_norm": 0.07997103035449982,
      "learning_rate": 4.986304738420684e-06,
      "logits/chosen": 14.172693252563477,
      "logits/rejected": 14.742494583129883,
      "logps/chosen": -0.28225988149642944,
      "logps/rejected": -0.34329456090927124,
      "loss": 0.9081,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.42338982224464417,
      "rewards/margins": 0.0915520042181015,
      "rewards/rejected": -0.5149418115615845,
      "step": 50
    },
    {
      "epoch": 0.04036326942482341,
      "eval_logits/chosen": 14.200166702270508,
      "eval_logits/rejected": 14.817726135253906,
      "eval_logps/chosen": -0.2625390887260437,
      "eval_logps/rejected": -0.3458769917488098,
      "eval_loss": 0.9080610275268555,
      "eval_rewards/accuracies": 0.5544554591178894,
      "eval_rewards/chosen": -0.39380866289138794,
      "eval_rewards/margins": 0.1250067949295044,
      "eval_rewards/rejected": -0.5188154578208923,
      "eval_runtime": 29.8098,
      "eval_samples_per_second": 26.87,
      "eval_steps_per_second": 3.388,
      "step": 50
    },
    {
      "epoch": 0.04843592330978809,
      "grad_norm": 0.06322095543146133,
      "learning_rate": 4.980286753286196e-06,
      "logits/chosen": 14.008010864257812,
      "logits/rejected": 14.975939750671387,
      "logps/chosen": -0.2697351574897766,
      "logps/rejected": -0.3445274233818054,
      "loss": 0.9195,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.40460270643234253,
      "rewards/margins": 0.1121884360909462,
      "rewards/rejected": -0.5167912244796753,
      "step": 60
    },
    {
      "epoch": 0.056508577194752774,
      "grad_norm": 0.12975476682186127,
      "learning_rate": 4.973180832407471e-06,
      "logits/chosen": 14.213134765625,
      "logits/rejected": 14.889978408813477,
      "logps/chosen": -0.2839818596839905,
      "logps/rejected": -0.3382417559623718,
      "loss": 0.9241,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.4259727895259857,
      "rewards/margins": 0.08138985931873322,
      "rewards/rejected": -0.5073626637458801,
      "step": 70
    },
    {
      "epoch": 0.06458123107971746,
      "grad_norm": 0.08352109789848328,
      "learning_rate": 4.964990092676263e-06,
      "logits/chosen": 14.395146369934082,
      "logits/rejected": 14.900177001953125,
      "logps/chosen": -0.27067264914512634,
      "logps/rejected": -0.3516673743724823,
      "loss": 0.9251,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.4060089588165283,
      "rewards/margins": 0.12149210274219513,
      "rewards/rejected": -0.5275009870529175,
      "step": 80
    },
    {
      "epoch": 0.07265388496468214,
      "grad_norm": 0.0793827548623085,
      "learning_rate": 4.9557181268217225e-06,
      "logits/chosen": 13.905145645141602,
      "logits/rejected": 14.897878646850586,
      "logps/chosen": -0.26814645528793335,
      "logps/rejected": -0.38061466813087463,
      "loss": 0.9055,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4022197127342224,
      "rewards/margins": 0.16870227456092834,
      "rewards/rejected": -0.5709219574928284,
      "step": 90
    },
    {
      "epoch": 0.08072653884964683,
      "grad_norm": 0.09212008118629456,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": 14.11627197265625,
      "logits/rejected": 14.571977615356445,
      "logps/chosen": -0.3130624294281006,
      "logps/rejected": -0.35073375701904297,
      "loss": 0.911,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.46959367394447327,
      "rewards/margins": 0.05650699883699417,
      "rewards/rejected": -0.5261006951332092,
      "step": 100
    },
    {
      "epoch": 0.08072653884964683,
      "eval_logits/chosen": 13.845876693725586,
      "eval_logits/rejected": 14.490789413452148,
      "eval_logps/chosen": -0.2531408667564392,
      "eval_logps/rejected": -0.3464036285877228,
      "eval_loss": 0.8986235857009888,
      "eval_rewards/accuracies": 0.5544554591178894,
      "eval_rewards/chosen": -0.3797112703323364,
      "eval_rewards/margins": 0.13989417254924774,
      "eval_rewards/rejected": -0.5196054577827454,
      "eval_runtime": 29.0914,
      "eval_samples_per_second": 27.534,
      "eval_steps_per_second": 3.472,
      "step": 100
    },
    {
      "epoch": 0.08879919273461151,
      "grad_norm": 0.08554862439632416,
      "learning_rate": 4.933947257182901e-06,
      "logits/chosen": 13.416229248046875,
      "logits/rejected": 14.582674026489258,
      "logps/chosen": -0.2523443102836609,
      "logps/rejected": -0.38751405477523804,
      "loss": 0.8993,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3785164952278137,
      "rewards/margins": 0.20275457203388214,
      "rewards/rejected": -0.5812710523605347,
      "step": 110
    },
    {
      "epoch": 0.09687184661957618,
      "grad_norm": 0.29209578037261963,
      "learning_rate": 4.921457902821578e-06,
      "logits/chosen": 13.874654769897461,
      "logits/rejected": 14.423624992370605,
      "logps/chosen": -0.27300480008125305,
      "logps/rejected": -0.3526575267314911,
      "loss": 0.8942,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.40950721502304077,
      "rewards/margins": 0.11947910487651825,
      "rewards/rejected": -0.5289863348007202,
      "step": 120
    },
    {
      "epoch": 0.10494450050454086,
      "grad_norm": 0.1028478816151619,
      "learning_rate": 4.907906416994146e-06,
      "logits/chosen": 14.030723571777344,
      "logits/rejected": 14.711235046386719,
      "logps/chosen": -0.27410784363746643,
      "logps/rejected": -0.3665519058704376,
      "loss": 0.8922,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.41116175055503845,
      "rewards/margins": 0.1386660784482956,
      "rewards/rejected": -0.5498278737068176,
      "step": 130
    },
    {
      "epoch": 0.11301715438950555,
      "grad_norm": 0.08459590375423431,
      "learning_rate": 4.893298743830168e-06,
      "logits/chosen": 13.477182388305664,
      "logits/rejected": 13.919464111328125,
      "logps/chosen": -0.2495063841342926,
      "logps/rejected": -0.3594816029071808,
      "loss": 0.8957,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.3742595613002777,
      "rewards/margins": 0.16496284306049347,
      "rewards/rejected": -0.5392224192619324,
      "step": 140
    },
    {
      "epoch": 0.12108980827447023,
      "grad_norm": 0.12467797845602036,
      "learning_rate": 4.8776412907378845e-06,
      "logits/chosen": 12.801656723022461,
      "logits/rejected": 13.564155578613281,
      "logps/chosen": -0.2779986262321472,
      "logps/rejected": -0.33566445112228394,
      "loss": 0.892,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4169979691505432,
      "rewards/margins": 0.08649872243404388,
      "rewards/rejected": -0.5034967064857483,
      "step": 150
    },
    {
      "epoch": 0.12108980827447023,
      "eval_logits/chosen": 12.169118881225586,
      "eval_logits/rejected": 13.019683837890625,
      "eval_logps/chosen": -0.25547870993614197,
      "eval_logps/rejected": -0.37277930974960327,
      "eval_loss": 0.8789658546447754,
      "eval_rewards/accuracies": 0.5742574334144592,
      "eval_rewards/chosen": -0.38321802020072937,
      "eval_rewards/margins": 0.17595094442367554,
      "eval_rewards/rejected": -0.5591689944267273,
      "eval_runtime": 29.0961,
      "eval_samples_per_second": 27.529,
      "eval_steps_per_second": 3.471,
      "step": 150
    },
    {
      "epoch": 0.12916246215943492,
      "grad_norm": 0.1712370663881302,
      "learning_rate": 4.860940925593703e-06,
      "logits/chosen": 11.972528457641602,
      "logits/rejected": 12.684088706970215,
      "logps/chosen": -0.28340521454811096,
      "logps/rejected": -0.3864063024520874,
      "loss": 0.8789,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.42510780692100525,
      "rewards/margins": 0.15450166165828705,
      "rewards/rejected": -0.5796095132827759,
      "step": 160
    },
    {
      "epoch": 0.13723511604439959,
      "grad_norm": 0.21750673651695251,
      "learning_rate": 4.84320497372973e-06,
      "logits/chosen": 11.93881607055664,
      "logits/rejected": 12.458230972290039,
      "logps/chosen": -0.274599552154541,
      "logps/rejected": -0.3882916271686554,
      "loss": 0.8857,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4118993282318115,
      "rewards/margins": 0.1705380380153656,
      "rewards/rejected": -0.5824374556541443,
      "step": 170
    },
    {
      "epoch": 0.14530776992936428,
      "grad_norm": 0.16000741720199585,
      "learning_rate": 4.824441214720629e-06,
      "logits/chosen": 9.631464958190918,
      "logits/rejected": 10.772969245910645,
      "logps/chosen": -0.287865549325943,
      "logps/rejected": -0.4817379415035248,
      "loss": 0.8699,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.4317983090877533,
      "rewards/margins": 0.29080861806869507,
      "rewards/rejected": -0.722606897354126,
      "step": 180
    },
    {
      "epoch": 0.15338042381432895,
      "grad_norm": 0.46293890476226807,
      "learning_rate": 4.804657878971252e-06,
      "logits/chosen": 9.081937789916992,
      "logits/rejected": 10.024572372436523,
      "logps/chosen": -0.2941994071006775,
      "logps/rejected": -0.4772109389305115,
      "loss": 0.8565,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.44129911065101624,
      "rewards/margins": 0.2745172679424286,
      "rewards/rejected": -0.7158163785934448,
      "step": 190
    },
    {
      "epoch": 0.16145307769929365,
      "grad_norm": 0.24658434092998505,
      "learning_rate": 4.783863644106502e-06,
      "logits/chosen": 7.767125606536865,
      "logits/rejected": 8.009145736694336,
      "logps/chosen": -0.32877305150032043,
      "logps/rejected": -0.47733697295188904,
      "loss": 0.8482,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.49315959215164185,
      "rewards/margins": 0.2228458821773529,
      "rewards/rejected": -0.7160054445266724,
      "step": 200
    },
    {
      "epoch": 0.16145307769929365,
      "eval_logits/chosen": 6.716187953948975,
      "eval_logits/rejected": 7.248146057128906,
      "eval_logps/chosen": -0.30652713775634766,
      "eval_logps/rejected": -0.5056277513504028,
      "eval_loss": 0.8243693113327026,
      "eval_rewards/accuracies": 0.603960394859314,
      "eval_rewards/chosen": -0.4597907066345215,
      "eval_rewards/margins": 0.2986510097980499,
      "eval_rewards/rejected": -0.7584417462348938,
      "eval_runtime": 29.0989,
      "eval_samples_per_second": 27.527,
      "eval_steps_per_second": 3.471,
      "step": 200
    },
    {
      "epoch": 0.16952573158425832,
      "grad_norm": 0.40796443819999695,
      "learning_rate": 4.762067631165049e-06,
      "logits/chosen": 5.932644844055176,
      "logits/rejected": 6.521953582763672,
      "logps/chosen": -0.32976508140563965,
      "logps/rejected": -0.5628186464309692,
      "loss": 0.7988,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.4946475923061371,
      "rewards/margins": 0.3495803475379944,
      "rewards/rejected": -0.8442279696464539,
      "step": 210
    },
    {
      "epoch": 0.17759838546922302,
      "grad_norm": 0.42468318343162537,
      "learning_rate": 4.7392794005985324e-06,
      "logits/chosen": 6.069305896759033,
      "logits/rejected": 5.8950395584106445,
      "logps/chosen": -0.37205421924591064,
      "logps/rejected": -0.6190425753593445,
      "loss": 0.796,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5580812692642212,
      "rewards/margins": 0.3704826235771179,
      "rewards/rejected": -0.9285639524459839,
      "step": 220
    },
    {
      "epoch": 0.1856710393541877,
      "grad_norm": 0.4138280153274536,
      "learning_rate": 4.715508948078037e-06,
      "logits/chosen": 3.5271706581115723,
      "logits/rejected": 3.363534927368164,
      "logps/chosen": -0.41677650809288025,
      "logps/rejected": -0.7121980786323547,
      "loss": 0.7457,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6251648664474487,
      "rewards/margins": 0.4431324005126953,
      "rewards/rejected": -1.0682971477508545,
      "step": 230
    },
    {
      "epoch": 0.19374369323915236,
      "grad_norm": 1.4327284097671509,
      "learning_rate": 4.690766700109659e-06,
      "logits/chosen": 2.296924114227295,
      "logits/rejected": 1.6135867834091187,
      "logps/chosen": -0.506227970123291,
      "logps/rejected": -0.8080593943595886,
      "loss": 0.7453,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.7593418955802917,
      "rewards/margins": 0.4527471661567688,
      "rewards/rejected": -1.21208918094635,
      "step": 240
    },
    {
      "epoch": 0.20181634712411706,
      "grad_norm": 0.4413074553012848,
      "learning_rate": 4.665063509461098e-06,
      "logits/chosen": 1.4450469017028809,
      "logits/rejected": 0.40727120637893677,
      "logps/chosen": -0.4749869406223297,
      "logps/rejected": -0.9967275857925415,
      "loss": 0.704,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.712480366230011,
      "rewards/margins": 0.7826108932495117,
      "rewards/rejected": -1.4950913190841675,
      "step": 250
    },
    {
      "epoch": 0.20181634712411706,
      "eval_logits/chosen": 1.4614256620407104,
      "eval_logits/rejected": 0.6895493865013123,
      "eval_logps/chosen": -0.5038881301879883,
      "eval_logps/rejected": -0.973581850528717,
      "eval_loss": 0.6982013583183289,
      "eval_rewards/accuracies": 0.6138613820075989,
      "eval_rewards/chosen": -0.7558321952819824,
      "eval_rewards/margins": 0.7045406699180603,
      "eval_rewards/rejected": -1.460372805595398,
      "eval_runtime": 29.0993,
      "eval_samples_per_second": 27.526,
      "eval_steps_per_second": 3.471,
      "step": 250
    },
    {
      "epoch": 0.20988900100908173,
      "grad_norm": 0.734251856803894,
      "learning_rate": 4.638410650401267e-06,
      "logits/chosen": 2.4274039268493652,
      "logits/rejected": 1.498230218887329,
      "logps/chosen": -0.5549699068069458,
      "logps/rejected": -0.9348627328872681,
      "loss": 0.7149,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.8324548602104187,
      "rewards/margins": 0.5698392987251282,
      "rewards/rejected": -1.4022941589355469,
      "step": 260
    },
    {
      "epoch": 0.21796165489404642,
      "grad_norm": 0.4445085823535919,
      "learning_rate": 4.610819813755038e-06,
      "logits/chosen": 2.042858600616455,
      "logits/rejected": 1.1695036888122559,
      "logps/chosen": -0.5573975443840027,
      "logps/rejected": -1.15065598487854,
      "loss": 0.6811,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8360962867736816,
      "rewards/margins": 0.889887809753418,
      "rewards/rejected": -1.72598397731781,
      "step": 270
    },
    {
      "epoch": 0.2260343087790111,
      "grad_norm": 0.5875476598739624,
      "learning_rate": 4.582303101775249e-06,
      "logits/chosen": 1.3029518127441406,
      "logits/rejected": 0.3808741271495819,
      "logps/chosen": -0.6656169891357422,
      "logps/rejected": -1.5305824279785156,
      "loss": 0.637,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9984253644943237,
      "rewards/margins": 1.2974482774734497,
      "rewards/rejected": -2.2958736419677734,
      "step": 280
    },
    {
      "epoch": 0.2341069626639758,
      "grad_norm": 0.3814420998096466,
      "learning_rate": 4.55287302283426e-06,
      "logits/chosen": 2.0200212001800537,
      "logits/rejected": 1.2681838274002075,
      "logps/chosen": -0.6776013970375061,
      "logps/rejected": -1.3369777202606201,
      "loss": 0.63,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.016402006149292,
      "rewards/margins": 0.9890643358230591,
      "rewards/rejected": -2.0054664611816406,
      "step": 290
    },
    {
      "epoch": 0.24217961654894046,
      "grad_norm": 0.6442322731018066,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": 1.6268196105957031,
      "logits/rejected": 0.4954712390899658,
      "logps/chosen": -0.8117648363113403,
      "logps/rejected": -1.7705228328704834,
      "loss": 0.6104,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2176473140716553,
      "rewards/margins": 1.4381370544433594,
      "rewards/rejected": -2.6557843685150146,
      "step": 300
    },
    {
      "epoch": 0.24217961654894046,
      "eval_logits/chosen": 1.4003607034683228,
      "eval_logits/rejected": 0.46628010272979736,
      "eval_logps/chosen": -0.7612115740776062,
      "eval_logps/rejected": -1.6895866394042969,
      "eval_loss": 0.5781419277191162,
      "eval_rewards/accuracies": 0.6534653306007385,
      "eval_rewards/chosen": -1.141817331314087,
      "eval_rewards/margins": 1.392562747001648,
      "eval_rewards/rejected": -2.5343799591064453,
      "eval_runtime": 29.102,
      "eval_samples_per_second": 27.524,
      "eval_steps_per_second": 3.471,
      "step": 300
    },
    {
      "epoch": 0.25025227043390513,
      "grad_norm": 0.7889758944511414,
      "learning_rate": 4.491324795060491e-06,
      "logits/chosen": 1.1702089309692383,
      "logits/rejected": 0.354276180267334,
      "logps/chosen": -0.8091352581977844,
      "logps/rejected": -1.944819450378418,
      "loss": 0.5807,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2137027978897095,
      "rewards/margins": 1.7035261392593384,
      "rewards/rejected": -2.9172286987304688,
      "step": 310
    },
    {
      "epoch": 0.25832492431886983,
      "grad_norm": 1.410145878791809,
      "learning_rate": 4.4592336433146e-06,
      "logits/chosen": 2.0981459617614746,
      "logits/rejected": 1.2289329767227173,
      "logps/chosen": -0.8151634931564331,
      "logps/rejected": -1.977259874343872,
      "loss": 0.5007,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.222745418548584,
      "rewards/margins": 1.7431443929672241,
      "rewards/rejected": -2.9658896923065186,
      "step": 320
    },
    {
      "epoch": 0.26639757820383453,
      "grad_norm": 0.5564689040184021,
      "learning_rate": 4.426283106939474e-06,
      "logits/chosen": 1.7921768426895142,
      "logits/rejected": 0.7705962061882019,
      "logps/chosen": -0.9244564771652222,
      "logps/rejected": -2.5274672508239746,
      "loss": 0.4757,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3866846561431885,
      "rewards/margins": 2.4045166969299316,
      "rewards/rejected": -3.791201114654541,
      "step": 330
    },
    {
      "epoch": 0.27447023208879917,
      "grad_norm": 0.7554243803024292,
      "learning_rate": 4.3924876391293915e-06,
      "logits/chosen": 1.947997808456421,
      "logits/rejected": 0.9661592245101929,
      "logps/chosen": -0.9154227375984192,
      "logps/rejected": -3.0766491889953613,
      "loss": 0.4581,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3731342554092407,
      "rewards/margins": 3.24183988571167,
      "rewards/rejected": -4.614974021911621,
      "step": 340
    },
    {
      "epoch": 0.28254288597376387,
      "grad_norm": 2.936426877975464,
      "learning_rate": 4.357862063693486e-06,
      "logits/chosen": 1.2846364974975586,
      "logits/rejected": 0.9968118667602539,
      "logps/chosen": -1.0272481441497803,
      "logps/rejected": -2.6980624198913574,
      "loss": 0.4989,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.54087233543396,
      "rewards/margins": 2.506221294403076,
      "rewards/rejected": -4.047093868255615,
      "step": 350
    },
    {
      "epoch": 0.28254288597376387,
      "eval_logits/chosen": 1.841333031654358,
      "eval_logits/rejected": 1.1651691198349,
      "eval_logps/chosen": -0.9644113183021545,
      "eval_logps/rejected": -2.631535053253174,
      "eval_loss": 0.44154587388038635,
      "eval_rewards/accuracies": 0.6732673048973083,
      "eval_rewards/chosen": -1.4466170072555542,
      "eval_rewards/margins": 2.500684976577759,
      "eval_rewards/rejected": -3.9473025798797607,
      "eval_runtime": 29.1062,
      "eval_samples_per_second": 27.52,
      "eval_steps_per_second": 3.47,
      "step": 350
    },
    {
      "epoch": 0.29061553985872857,
      "grad_norm": 0.49919044971466064,
      "learning_rate": 4.322421568553529e-06,
      "logits/chosen": 1.9874728918075562,
      "logits/rejected": 1.464611291885376,
      "logps/chosen": -0.9963932037353516,
      "logps/rejected": -2.8804218769073486,
      "loss": 0.4373,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4945898056030273,
      "rewards/margins": 2.8260433673858643,
      "rewards/rejected": -4.320633411407471,
      "step": 360
    },
    {
      "epoch": 0.29868819374369326,
      "grad_norm": 1.4991668462753296,
      "learning_rate": 4.286181699082008e-06,
      "logits/chosen": 2.5421957969665527,
      "logits/rejected": 1.941922903060913,
      "logps/chosen": -1.0214306116104126,
      "logps/rejected": -3.2893283367156982,
      "loss": 0.438,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5321458578109741,
      "rewards/margins": 3.4018466472625732,
      "rewards/rejected": -4.933992862701416,
      "step": 370
    },
    {
      "epoch": 0.3067608476286579,
      "grad_norm": 1.933100938796997,
      "learning_rate": 4.249158351283414e-06,
      "logits/chosen": 1.80439031124115,
      "logits/rejected": 1.4421275854110718,
      "logps/chosen": -1.1270357370376587,
      "logps/rejected": -3.17500901222229,
      "loss": 0.4542,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.6905533075332642,
      "rewards/margins": 3.071959972381592,
      "rewards/rejected": -4.762513160705566,
      "step": 380
    },
    {
      "epoch": 0.3148335015136226,
      "grad_norm": 3.017254590988159,
      "learning_rate": 4.211367764821722e-06,
      "logits/chosen": 3.4090209007263184,
      "logits/rejected": 2.784639835357666,
      "logps/chosen": -1.1713608503341675,
      "logps/rejected": -3.326244831085205,
      "loss": 0.4816,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.757041335105896,
      "rewards/margins": 3.232325792312622,
      "rewards/rejected": -4.9893670082092285,
      "step": 390
    },
    {
      "epoch": 0.3229061553985873,
      "grad_norm": 0.5897337198257446,
      "learning_rate": 4.172826515897146e-06,
      "logits/chosen": 3.6071503162384033,
      "logits/rejected": 2.738395929336548,
      "logps/chosen": -1.133429765701294,
      "logps/rejected": -3.509474277496338,
      "loss": 0.4431,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.7001447677612305,
      "rewards/margins": 3.5640671253204346,
      "rewards/rejected": -5.264212131500244,
      "step": 400
    },
    {
      "epoch": 0.3229061553985873,
      "eval_logits/chosen": 2.893691062927246,
      "eval_logits/rejected": 2.259718656539917,
      "eval_logps/chosen": -1.0705492496490479,
      "eval_logps/rejected": -2.935904026031494,
      "eval_loss": 0.4156961143016815,
      "eval_rewards/accuracies": 0.6732673048973083,
      "eval_rewards/chosen": -1.6058237552642822,
      "eval_rewards/margins": 2.798032283782959,
      "eval_rewards/rejected": -4.403855800628662,
      "eval_runtime": 29.186,
      "eval_samples_per_second": 27.445,
      "eval_steps_per_second": 3.461,
      "step": 400
    },
    {
      "epoch": 0.33097880928355194,
      "grad_norm": 2.219709634780884,
      "learning_rate": 4.133551509975264e-06,
      "logits/chosen": 2.8940415382385254,
      "logits/rejected": 2.545257091522217,
      "logps/chosen": -1.1774877309799194,
      "logps/rejected": -3.4022269248962402,
      "loss": 0.427,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7662317752838135,
      "rewards/margins": 3.3371081352233887,
      "rewards/rejected": -5.1033406257629395,
      "step": 410
    },
    {
      "epoch": 0.33905146316851664,
      "grad_norm": 0.9891649484634399,
      "learning_rate": 4.093559974371725e-06,
      "logits/chosen": 3.180427074432373,
      "logits/rejected": 2.8354597091674805,
      "logps/chosen": -1.2984545230865479,
      "logps/rejected": -3.4927687644958496,
      "loss": 0.4384,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9476817846298218,
      "rewards/margins": 3.291471481323242,
      "rewards/rejected": -5.2391533851623535,
      "step": 420
    },
    {
      "epoch": 0.34712411705348134,
      "grad_norm": 0.4516288936138153,
      "learning_rate": 4.052869450695776e-06,
      "logits/chosen": 4.026360511779785,
      "logits/rejected": 3.3932456970214844,
      "logps/chosen": -1.1195746660232544,
      "logps/rejected": -3.4704582691192627,
      "loss": 0.3934,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.6793619394302368,
      "rewards/margins": 3.5263259410858154,
      "rewards/rejected": -5.205687522888184,
      "step": 430
    },
    {
      "epoch": 0.35519677093844604,
      "grad_norm": 0.5276560187339783,
      "learning_rate": 4.011497787155938e-06,
      "logits/chosen": 3.410902500152588,
      "logits/rejected": 2.906996965408325,
      "logps/chosen": -1.205904245376587,
      "logps/rejected": -3.7434115409851074,
      "loss": 0.4147,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.8088566064834595,
      "rewards/margins": 3.806260585784912,
      "rewards/rejected": -5.615117073059082,
      "step": 440
    },
    {
      "epoch": 0.3632694248234107,
      "grad_norm": 0.4435446560382843,
      "learning_rate": 3.969463130731183e-06,
      "logits/chosen": 3.7616615295410156,
      "logits/rejected": 3.4183363914489746,
      "logps/chosen": -1.2180852890014648,
      "logps/rejected": -3.865541934967041,
      "loss": 0.4354,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8271278142929077,
      "rewards/margins": 3.9711856842041016,
      "rewards/rejected": -5.798312664031982,
      "step": 450
    },
    {
      "epoch": 0.3632694248234107,
      "eval_logits/chosen": 3.55381178855896,
      "eval_logits/rejected": 3.01943039894104,
      "eval_logps/chosen": -1.3239827156066895,
      "eval_logps/rejected": -3.6477084159851074,
      "eval_loss": 0.3905698359012604,
      "eval_rewards/accuracies": 0.7029703259468079,
      "eval_rewards/chosen": -1.9859741926193237,
      "eval_rewards/margins": 3.485589027404785,
      "eval_rewards/rejected": -5.471563816070557,
      "eval_runtime": 29.0958,
      "eval_samples_per_second": 27.53,
      "eval_steps_per_second": 3.471,
      "step": 450
    },
    {
      "epoch": 0.3713420787083754,
      "grad_norm": 0.8419796824455261,
      "learning_rate": 3.92678391921108e-06,
      "logits/chosen": 3.3496861457824707,
      "logits/rejected": 3.235544204711914,
      "logps/chosen": -1.4899990558624268,
      "logps/rejected": -4.507351398468018,
      "loss": 0.4023,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.2349984645843506,
      "rewards/margins": 4.526029109954834,
      "rewards/rejected": -6.7610273361206055,
      "step": 460
    },
    {
      "epoch": 0.3794147325933401,
      "grad_norm": 1.2379052639007568,
      "learning_rate": 3.88347887310836e-06,
      "logits/chosen": 3.5375685691833496,
      "logits/rejected": 3.34586763381958,
      "logps/chosen": -1.5865886211395264,
      "logps/rejected": -4.887473106384277,
      "loss": 0.3979,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.379883289337158,
      "rewards/margins": 4.9513258934021,
      "rewards/rejected": -7.331210136413574,
      "step": 470
    },
    {
      "epoch": 0.3874873864783047,
      "grad_norm": 1.9969083070755005,
      "learning_rate": 3.839566987447492e-06,
      "logits/chosen": 3.3116583824157715,
      "logits/rejected": 2.9652373790740967,
      "logps/chosen": -2.0900139808654785,
      "logps/rejected": -4.921124458312988,
      "loss": 0.3356,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.135021209716797,
      "rewards/margins": 4.246665000915527,
      "rewards/rejected": -7.381686210632324,
      "step": 480
    },
    {
      "epoch": 0.3955600403632694,
      "grad_norm": 2.3405466079711914,
      "learning_rate": 3.795067523432826e-06,
      "logits/chosen": 2.542064666748047,
      "logits/rejected": 2.503020763397217,
      "logps/chosen": -2.4674277305603027,
      "logps/rejected": -5.703408718109131,
      "loss": 0.3193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.701141834259033,
      "rewards/margins": 4.853971004486084,
      "rewards/rejected": -8.555112838745117,
      "step": 490
    },
    {
      "epoch": 0.4036326942482341,
      "grad_norm": 1.4798164367675781,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": 3.79949688911438,
      "logits/rejected": 3.7085328102111816,
      "logps/chosen": -2.6355605125427246,
      "logps/rejected": -5.005209922790527,
      "loss": 0.3103,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.953340530395508,
      "rewards/margins": 3.554474353790283,
      "rewards/rejected": -7.507814884185791,
      "step": 500
    },
    {
      "epoch": 0.4036326942482341,
      "eval_logits/chosen": 3.31948184967041,
      "eval_logits/rejected": 2.9104762077331543,
      "eval_logps/chosen": -2.606774091720581,
      "eval_logps/rejected": -5.2280120849609375,
      "eval_loss": 0.3228474259376526,
      "eval_rewards/accuracies": 0.9306930899620056,
      "eval_rewards/chosen": -3.9101614952087402,
      "eval_rewards/margins": 3.931856870651245,
      "eval_rewards/rejected": -7.8420186042785645,
      "eval_runtime": 29.0975,
      "eval_samples_per_second": 27.528,
      "eval_steps_per_second": 3.471,
      "step": 500
    },
    {
      "epoch": 0.4117053481331988,
      "grad_norm": 1.4098985195159912,
      "learning_rate": 3.7043841852542884e-06,
      "logits/chosen": 2.69830584526062,
      "logits/rejected": 2.7101688385009766,
      "logps/chosen": -2.7831828594207764,
      "logps/rejected": -5.963825225830078,
      "loss": 0.2884,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.174774169921875,
      "rewards/margins": 4.7709641456604,
      "rewards/rejected": -8.945737838745117,
      "step": 510
    },
    {
      "epoch": 0.41977800201816345,
      "grad_norm": 3.457231283187866,
      "learning_rate": 3.658240087799655e-06,
      "logits/chosen": 3.335204601287842,
      "logits/rejected": 3.1935055255889893,
      "logps/chosen": -2.66495943069458,
      "logps/rejected": -5.2761640548706055,
      "loss": 0.3093,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.997438907623291,
      "rewards/margins": 3.916806697845459,
      "rewards/rejected": -7.91424560546875,
      "step": 520
    },
    {
      "epoch": 0.42785065590312815,
      "grad_norm": 1.7744654417037964,
      "learning_rate": 3.611587947962319e-06,
      "logits/chosen": 2.725924015045166,
      "logits/rejected": 2.605053186416626,
      "logps/chosen": -2.5286645889282227,
      "logps/rejected": -5.774144172668457,
      "loss": 0.2972,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.792996883392334,
      "rewards/margins": 4.868220329284668,
      "rewards/rejected": -8.661216735839844,
      "step": 530
    },
    {
      "epoch": 0.43592330978809285,
      "grad_norm": 2.863692283630371,
      "learning_rate": 3.564448228912682e-06,
      "logits/chosen": 2.304443120956421,
      "logits/rejected": 2.4836020469665527,
      "logps/chosen": -2.699148654937744,
      "logps/rejected": -6.340291500091553,
      "loss": 0.3136,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.048723220825195,
      "rewards/margins": 5.461714267730713,
      "rewards/rejected": -9.510437965393066,
      "step": 540
    },
    {
      "epoch": 0.4439959636730575,
      "grad_norm": 4.6681623458862305,
      "learning_rate": 3.516841607689501e-06,
      "logits/chosen": 3.1207594871520996,
      "logits/rejected": 2.947653293609619,
      "logps/chosen": -3.0062317848205566,
      "logps/rejected": -6.26816463470459,
      "loss": 0.2822,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.509347438812256,
      "rewards/margins": 4.892899513244629,
      "rewards/rejected": -9.402246475219727,
      "step": 550
    },
    {
      "epoch": 0.4439959636730575,
      "eval_logits/chosen": 2.879721164703369,
      "eval_logits/rejected": 2.52551007270813,
      "eval_logps/chosen": -2.6468751430511475,
      "eval_logps/rejected": -5.636789321899414,
      "eval_loss": 0.2939932942390442,
      "eval_rewards/accuracies": 0.9306930899620056,
      "eval_rewards/chosen": -3.9703128337860107,
      "eval_rewards/margins": 4.4848713874816895,
      "eval_rewards/rejected": -8.455184936523438,
      "eval_runtime": 29.4025,
      "eval_samples_per_second": 27.243,
      "eval_steps_per_second": 3.435,
      "step": 550
    },
    {
      "epoch": 0.4520686175580222,
      "grad_norm": 1.9975569248199463,
      "learning_rate": 3.4687889661302577e-06,
      "logits/chosen": 2.5885415077209473,
      "logits/rejected": 2.5374302864074707,
      "logps/chosen": -2.886643171310425,
      "logps/rejected": -6.636415004730225,
      "loss": 0.2736,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.329964637756348,
      "rewards/margins": 5.624656677246094,
      "rewards/rejected": -9.954621315002441,
      "step": 560
    },
    {
      "epoch": 0.4601412714429869,
      "grad_norm": 3.3965606689453125,
      "learning_rate": 3.4203113817116955e-06,
      "logits/chosen": 2.0093374252319336,
      "logits/rejected": 2.1064200401306152,
      "logps/chosen": -3.251981735229492,
      "logps/rejected": -6.749331474304199,
      "loss": 0.2776,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.877972602844238,
      "rewards/margins": 5.246025085449219,
      "rewards/rejected": -10.12399673461914,
      "step": 570
    },
    {
      "epoch": 0.4682139253279516,
      "grad_norm": 2.7079927921295166,
      "learning_rate": 3.3714301183045382e-06,
      "logits/chosen": 3.916841506958008,
      "logits/rejected": 3.524440288543701,
      "logps/chosen": -3.304469347000122,
      "logps/rejected": -7.1993279457092285,
      "loss": 0.2547,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.956704139709473,
      "rewards/margins": 5.842287540435791,
      "rewards/rejected": -10.798992156982422,
      "step": 580
    },
    {
      "epoch": 0.47628657921291623,
      "grad_norm": 7.504203796386719,
      "learning_rate": 3.3221666168464584e-06,
      "logits/chosen": 2.7350666522979736,
      "logits/rejected": 2.2590534687042236,
      "logps/chosen": -3.0954322814941406,
      "logps/rejected": -6.782465934753418,
      "loss": 0.2839,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.643147945404053,
      "rewards/margins": 5.530551433563232,
      "rewards/rejected": -10.173700332641602,
      "step": 590
    },
    {
      "epoch": 0.4843592330978809,
      "grad_norm": 2.8209569454193115,
      "learning_rate": 3.272542485937369e-06,
      "logits/chosen": 3.7743606567382812,
      "logits/rejected": 3.2480530738830566,
      "logps/chosen": -3.196892261505127,
      "logps/rejected": -6.42517614364624,
      "loss": 0.3237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.7953386306762695,
      "rewards/margins": 4.8424248695373535,
      "rewards/rejected": -9.637763977050781,
      "step": 600
    },
    {
      "epoch": 0.4843592330978809,
      "eval_logits/chosen": 2.6344597339630127,
      "eval_logits/rejected": 2.46531081199646,
      "eval_logps/chosen": -2.9877941608428955,
      "eval_logps/rejected": -6.266376972198486,
      "eval_loss": 0.2827170193195343,
      "eval_rewards/accuracies": 0.9603960514068604,
      "eval_rewards/chosen": -4.481690406799316,
      "eval_rewards/margins": 4.917874336242676,
      "eval_rewards/rejected": -9.399564743041992,
      "eval_runtime": 29.0997,
      "eval_samples_per_second": 27.526,
      "eval_steps_per_second": 3.471,
      "step": 600
    },
    {
      "epoch": 0.4924318869828456,
      "grad_norm": 7.072656631469727,
      "learning_rate": 3.222579492361179e-06,
      "logits/chosen": 3.2776103019714355,
      "logits/rejected": 3.166206121444702,
      "logps/chosen": -3.193716049194336,
      "logps/rejected": -6.185642242431641,
      "loss": 0.2986,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.790574073791504,
      "rewards/margins": 4.487889289855957,
      "rewards/rejected": -9.278463363647461,
      "step": 610
    },
    {
      "epoch": 0.5005045408678103,
      "grad_norm": 2.7188704013824463,
      "learning_rate": 3.1722995515381644e-06,
      "logits/chosen": 3.189532518386841,
      "logits/rejected": 2.9885735511779785,
      "logps/chosen": -3.3461785316467285,
      "logps/rejected": -6.720322608947754,
      "loss": 0.2776,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.019268035888672,
      "rewards/margins": 5.061214447021484,
      "rewards/rejected": -10.080483436584473,
      "step": 620
    },
    {
      "epoch": 0.508577194752775,
      "grad_norm": 2.6990387439727783,
      "learning_rate": 3.121724717912138e-06,
      "logits/chosen": 3.7751071453094482,
      "logits/rejected": 3.406925916671753,
      "logps/chosen": -3.3688464164733887,
      "logps/rejected": -6.132623672485352,
      "loss": 0.2688,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -5.053269386291504,
      "rewards/margins": 4.145665168762207,
      "rewards/rejected": -9.198934555053711,
      "step": 630
    },
    {
      "epoch": 0.5166498486377397,
      "grad_norm": 1.964301347732544,
      "learning_rate": 3.0708771752766397e-06,
      "logits/chosen": 2.6602158546447754,
      "logits/rejected": 2.162348508834839,
      "logps/chosen": -2.884718894958496,
      "logps/rejected": -6.869751930236816,
      "loss": 0.2435,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.327078342437744,
      "rewards/margins": 5.9775495529174805,
      "rewards/rejected": -10.304628372192383,
      "step": 640
    },
    {
      "epoch": 0.5247225025227044,
      "grad_norm": 3.201704740524292,
      "learning_rate": 3.019779227044398e-06,
      "logits/chosen": 3.83849835395813,
      "logits/rejected": 3.5917961597442627,
      "logps/chosen": -3.2030792236328125,
      "logps/rejected": -6.711284637451172,
      "loss": 0.2865,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.804618835449219,
      "rewards/margins": 5.262308120727539,
      "rewards/rejected": -10.066926956176758,
      "step": 650
    },
    {
      "epoch": 0.5247225025227044,
      "eval_logits/chosen": 2.6539251804351807,
      "eval_logits/rejected": 2.3508856296539307,
      "eval_logps/chosen": -3.0629732608795166,
      "eval_logps/rejected": -6.429784297943115,
      "eval_loss": 0.25649499893188477,
      "eval_rewards/accuracies": 0.9702970385551453,
      "eval_rewards/chosen": -4.594459533691406,
      "eval_rewards/margins": 5.0502166748046875,
      "eval_rewards/rejected": -9.644676208496094,
      "eval_runtime": 29.1031,
      "eval_samples_per_second": 27.523,
      "eval_steps_per_second": 3.47,
      "step": 650
    },
    {
      "epoch": 0.5327951564076691,
      "grad_norm": 3.517667055130005,
      "learning_rate": 2.9684532864643123e-06,
      "logits/chosen": 2.804810047149658,
      "logits/rejected": 2.8151614665985107,
      "logps/chosen": -3.4792568683624268,
      "logps/rejected": -7.08618688583374,
      "loss": 0.243,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.2188849449157715,
      "rewards/margins": 5.41039514541626,
      "rewards/rejected": -10.629280090332031,
      "step": 660
    },
    {
      "epoch": 0.5408678102926338,
      "grad_norm": 8.366011619567871,
      "learning_rate": 2.9169218667902562e-06,
      "logits/chosen": 2.1079938411712646,
      "logits/rejected": 2.2689383029937744,
      "logps/chosen": -3.3151912689208984,
      "logps/rejected": -7.454249382019043,
      "loss": 0.266,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.972787380218506,
      "rewards/margins": 6.208587169647217,
      "rewards/rejected": -11.181374549865723,
      "step": 670
    },
    {
      "epoch": 0.5489404641775983,
      "grad_norm": 4.161479473114014,
      "learning_rate": 2.8652075714060296e-06,
      "logits/chosen": 2.1858832836151123,
      "logits/rejected": 2.1688101291656494,
      "logps/chosen": -3.3753318786621094,
      "logps/rejected": -6.9990692138671875,
      "loss": 0.2356,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.0629987716674805,
      "rewards/margins": 5.435605525970459,
      "rewards/rejected": -10.498603820800781,
      "step": 680
    },
    {
      "epoch": 0.557013118062563,
      "grad_norm": 2.156621217727661,
      "learning_rate": 2.813333083910761e-06,
      "logits/chosen": 2.5004096031188965,
      "logits/rejected": 2.289215087890625,
      "logps/chosen": -3.711970567703247,
      "logps/rejected": -7.685518741607666,
      "loss": 0.2286,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.56795597076416,
      "rewards/margins": 5.960321426391602,
      "rewards/rejected": -11.528276443481445,
      "step": 690
    },
    {
      "epoch": 0.5650857719475277,
      "grad_norm": 4.547532081604004,
      "learning_rate": 2.761321158169134e-06,
      "logits/chosen": 3.112332820892334,
      "logits/rejected": 2.685454845428467,
      "logps/chosen": -3.510646343231201,
      "logps/rejected": -7.8125505447387695,
      "loss": 0.2222,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.265969276428223,
      "rewards/margins": 6.452856540679932,
      "rewards/rejected": -11.718826293945312,
      "step": 700
    },
    {
      "epoch": 0.5650857719475277,
      "eval_logits/chosen": 2.7667236328125,
      "eval_logits/rejected": 2.4983408451080322,
      "eval_logps/chosen": -3.292154550552368,
      "eval_logps/rejected": -6.814427375793457,
      "eval_loss": 0.24892139434814453,
      "eval_rewards/accuracies": 0.9702970385551453,
      "eval_rewards/chosen": -4.938231468200684,
      "eval_rewards/margins": 5.2834086418151855,
      "eval_rewards/rejected": -10.221639633178711,
      "eval_runtime": 29.0862,
      "eval_samples_per_second": 27.539,
      "eval_steps_per_second": 3.472,
      "step": 700
    },
    {
      "epoch": 0.5731584258324924,
      "grad_norm": 3.3842532634735107,
      "learning_rate": 2.70919460833079e-06,
      "logits/chosen": 2.6863622665405273,
      "logits/rejected": 2.7381534576416016,
      "logps/chosen": -3.1926321983337402,
      "logps/rejected": -7.823538780212402,
      "loss": 0.2153,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.788949012756348,
      "rewards/margins": 6.946360111236572,
      "rewards/rejected": -11.735308647155762,
      "step": 710
    },
    {
      "epoch": 0.5812310797174571,
      "grad_norm": 2.0056378841400146,
      "learning_rate": 2.6569762988232838e-06,
      "logits/chosen": 2.938141345977783,
      "logits/rejected": 2.8052525520324707,
      "logps/chosen": -3.4339518547058105,
      "logps/rejected": -7.353487968444824,
      "loss": 0.2422,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.150927543640137,
      "rewards/margins": 5.879303932189941,
      "rewards/rejected": -11.030232429504395,
      "step": 720
    },
    {
      "epoch": 0.5893037336024218,
      "grad_norm": 2.9281482696533203,
      "learning_rate": 2.604689134322999e-06,
      "logits/chosen": 3.5540454387664795,
      "logits/rejected": 3.6734695434570312,
      "logps/chosen": -3.0852925777435303,
      "logps/rejected": -6.869669437408447,
      "loss": 0.2694,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.627939224243164,
      "rewards/margins": 5.676565647125244,
      "rewards/rejected": -10.304505348205566,
      "step": 730
    },
    {
      "epoch": 0.5973763874873865,
      "grad_norm": 3.3841099739074707,
      "learning_rate": 2.5523560497083927e-06,
      "logits/chosen": 2.7171874046325684,
      "logits/rejected": 2.4911468029022217,
      "logps/chosen": -3.4943466186523438,
      "logps/rejected": -7.486982822418213,
      "loss": 0.2184,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.241519927978516,
      "rewards/margins": 5.988953113555908,
      "rewards/rejected": -11.230472564697266,
      "step": 740
    },
    {
      "epoch": 0.6054490413723511,
      "grad_norm": 2.6915197372436523,
      "learning_rate": 2.5e-06,
      "logits/chosen": 2.866816759109497,
      "logits/rejected": 2.84656023979187,
      "logps/chosen": -3.155074119567871,
      "logps/rejected": -8.019740104675293,
      "loss": 0.2364,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.732610702514648,
      "rewards/margins": 7.297000885009766,
      "rewards/rejected": -12.029611587524414,
      "step": 750
    },
    {
      "epoch": 0.6054490413723511,
      "eval_logits/chosen": 2.7122883796691895,
      "eval_logits/rejected": 2.4663355350494385,
      "eval_logps/chosen": -3.362666606903076,
      "eval_logps/rejected": -7.107527732849121,
      "eval_loss": 0.2374112904071808,
      "eval_rewards/accuracies": 0.9801980257034302,
      "eval_rewards/chosen": -5.043999671936035,
      "eval_rewards/margins": 5.617292881011963,
      "eval_rewards/rejected": -10.661293983459473,
      "eval_runtime": 29.0963,
      "eval_samples_per_second": 27.529,
      "eval_steps_per_second": 3.471,
      "step": 750
    },
    {
      "epoch": 0.6135216952573158,
      "grad_norm": 5.086276531219482,
      "learning_rate": 2.447643950291608e-06,
      "logits/chosen": 3.0712199211120605,
      "logits/rejected": 2.7986550331115723,
      "logps/chosen": -3.759767532348633,
      "logps/rejected": -8.20389175415039,
      "loss": 0.2467,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.639651298522949,
      "rewards/margins": 6.666187286376953,
      "rewards/rejected": -12.305837631225586,
      "step": 760
    },
    {
      "epoch": 0.6215943491422805,
      "grad_norm": 4.272627830505371,
      "learning_rate": 2.3953108656770018e-06,
      "logits/chosen": 2.4291179180145264,
      "logits/rejected": 2.247910737991333,
      "logps/chosen": -3.4369187355041504,
      "logps/rejected": -8.507123947143555,
      "loss": 0.2287,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.155378341674805,
      "rewards/margins": 7.605309963226318,
      "rewards/rejected": -12.760688781738281,
      "step": 770
    },
    {
      "epoch": 0.6296670030272452,
      "grad_norm": 4.545230388641357,
      "learning_rate": 2.3430237011767166e-06,
      "logits/chosen": 3.324824810028076,
      "logits/rejected": 3.0623250007629395,
      "logps/chosen": -3.5336334705352783,
      "logps/rejected": -7.56264591217041,
      "loss": 0.259,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.300450325012207,
      "rewards/margins": 6.043517589569092,
      "rewards/rejected": -11.34396743774414,
      "step": 780
    },
    {
      "epoch": 0.6377396569122099,
      "grad_norm": 3.5774872303009033,
      "learning_rate": 2.290805391669212e-06,
      "logits/chosen": 2.70954966545105,
      "logits/rejected": 2.559739589691162,
      "logps/chosen": -3.2724106311798096,
      "logps/rejected": -7.964414119720459,
      "loss": 0.243,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.908616065979004,
      "rewards/margins": 7.0380048751831055,
      "rewards/rejected": -11.94662094116211,
      "step": 790
    },
    {
      "epoch": 0.6458123107971746,
      "grad_norm": 3.9293906688690186,
      "learning_rate": 2.238678841830867e-06,
      "logits/chosen": 2.2065200805664062,
      "logits/rejected": 2.5098512172698975,
      "logps/chosen": -3.324014186859131,
      "logps/rejected": -8.072491645812988,
      "loss": 0.2752,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.986021518707275,
      "rewards/margins": 7.122715950012207,
      "rewards/rejected": -12.108736038208008,
      "step": 800
    },
    {
      "epoch": 0.6458123107971746,
      "eval_logits/chosen": 2.7405736446380615,
      "eval_logits/rejected": 2.500715732574463,
      "eval_logps/chosen": -3.4503068923950195,
      "eval_logps/rejected": -7.3543477058410645,
      "eval_loss": 0.22608846426010132,
      "eval_rewards/accuracies": 0.9801980257034302,
      "eval_rewards/chosen": -5.175460338592529,
      "eval_rewards/margins": 5.856060981750488,
      "eval_rewards/rejected": -11.031521797180176,
      "eval_runtime": 29.0926,
      "eval_samples_per_second": 27.533,
      "eval_steps_per_second": 3.472,
      "step": 800
    },
    {
      "epoch": 0.6538849646821393,
      "grad_norm": 3.1484081745147705,
      "learning_rate": 2.186666916089239e-06,
      "logits/chosen": 2.620433807373047,
      "logits/rejected": 2.554230213165283,
      "logps/chosen": -3.826986312866211,
      "logps/rejected": -8.595190048217773,
      "loss": 0.2267,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.740479469299316,
      "rewards/margins": 7.152307033538818,
      "rewards/rejected": -12.892786026000977,
      "step": 810
    },
    {
      "epoch": 0.6619576185671039,
      "grad_norm": 3.712259531021118,
      "learning_rate": 2.134792428593971e-06,
      "logits/chosen": 3.1829707622528076,
      "logits/rejected": 3.0434184074401855,
      "logps/chosen": -3.3385536670684814,
      "logps/rejected": -7.571812629699707,
      "loss": 0.2079,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.0078301429748535,
      "rewards/margins": 6.349888801574707,
      "rewards/rejected": -11.357718467712402,
      "step": 820
    },
    {
      "epoch": 0.6700302724520686,
      "grad_norm": 5.120253086090088,
      "learning_rate": 2.0830781332097446e-06,
      "logits/chosen": 3.631582736968994,
      "logits/rejected": 3.769865036010742,
      "logps/chosen": -3.2808995246887207,
      "logps/rejected": -7.530601501464844,
      "loss": 0.2213,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.92134952545166,
      "rewards/margins": 6.3745527267456055,
      "rewards/rejected": -11.29590129852295,
      "step": 830
    },
    {
      "epoch": 0.6781029263370333,
      "grad_norm": 3.723133087158203,
      "learning_rate": 2.031546713535688e-06,
      "logits/chosen": 2.7661056518554688,
      "logits/rejected": 2.7924928665161133,
      "logps/chosen": -3.5595524311065674,
      "logps/rejected": -7.967140197753906,
      "loss": 0.2381,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.339327812194824,
      "rewards/margins": 6.611382961273193,
      "rewards/rejected": -11.950711250305176,
      "step": 840
    },
    {
      "epoch": 0.686175580221998,
      "grad_norm": 3.4145116806030273,
      "learning_rate": 1.9802207729556023e-06,
      "logits/chosen": 2.4502460956573486,
      "logits/rejected": 2.364321708679199,
      "logps/chosen": -3.2101433277130127,
      "logps/rejected": -8.211054801940918,
      "loss": 0.2112,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.81521463394165,
      "rewards/margins": 7.501368522644043,
      "rewards/rejected": -12.316584587097168,
      "step": 850
    },
    {
      "epoch": 0.686175580221998,
      "eval_logits/chosen": 2.7086803913116455,
      "eval_logits/rejected": 2.437659978866577,
      "eval_logps/chosen": -3.123488664627075,
      "eval_logps/rejected": -7.074939727783203,
      "eval_loss": 0.22627003490924835,
      "eval_rewards/accuracies": 0.9702970385551453,
      "eval_rewards/chosen": -4.685232639312744,
      "eval_rewards/margins": 5.927176475524902,
      "eval_rewards/rejected": -10.612408638000488,
      "eval_runtime": 29.0746,
      "eval_samples_per_second": 27.55,
      "eval_steps_per_second": 3.474,
      "step": 850
    },
    {
      "epoch": 0.6942482341069627,
      "grad_norm": 3.628746509552002,
      "learning_rate": 1.9291228247233607e-06,
      "logits/chosen": 2.9090940952301025,
      "logits/rejected": 2.9559710025787354,
      "logps/chosen": -3.447845458984375,
      "logps/rejected": -7.511412620544434,
      "loss": 0.2154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.171767711639404,
      "rewards/margins": 6.09535026550293,
      "rewards/rejected": -11.267118453979492,
      "step": 860
    },
    {
      "epoch": 0.7023208879919274,
      "grad_norm": 3.231868267059326,
      "learning_rate": 1.8782752820878636e-06,
      "logits/chosen": 2.6312456130981445,
      "logits/rejected": 2.458289861679077,
      "logps/chosen": -3.6256003379821777,
      "logps/rejected": -8.079771041870117,
      "loss": 0.2386,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.438401699066162,
      "rewards/margins": 6.681253910064697,
      "rewards/rejected": -12.11965560913086,
      "step": 870
    },
    {
      "epoch": 0.7103935418768921,
      "grad_norm": 5.360827445983887,
      "learning_rate": 1.827700448461836e-06,
      "logits/chosen": 2.383152961730957,
      "logits/rejected": 2.461998701095581,
      "logps/chosen": -3.555072069168091,
      "logps/rejected": -8.438495635986328,
      "loss": 0.2126,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.332608222961426,
      "rewards/margins": 7.32513427734375,
      "rewards/rejected": -12.657742500305176,
      "step": 880
    },
    {
      "epoch": 0.7184661957618567,
      "grad_norm": 5.990912914276123,
      "learning_rate": 1.7774205076388207e-06,
      "logits/chosen": 3.3776676654815674,
      "logits/rejected": 2.913318157196045,
      "logps/chosen": -3.9117507934570312,
      "logps/rejected": -8.53381633758545,
      "loss": 0.185,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.8676252365112305,
      "rewards/margins": 6.933099269866943,
      "rewards/rejected": -12.800724983215332,
      "step": 890
    },
    {
      "epoch": 0.7265388496468214,
      "grad_norm": 6.092798233032227,
      "learning_rate": 1.7274575140626318e-06,
      "logits/chosen": 1.917933702468872,
      "logits/rejected": 1.6799787282943726,
      "logps/chosen": -3.4144363403320312,
      "logps/rejected": -8.597683906555176,
      "loss": 0.2383,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.121654987335205,
      "rewards/margins": 7.7748703956604,
      "rewards/rejected": -12.896525382995605,
      "step": 900
    },
    {
      "epoch": 0.7265388496468214,
      "eval_logits/chosen": 2.6657638549804688,
      "eval_logits/rejected": 2.4586844444274902,
      "eval_logps/chosen": -3.4474172592163086,
      "eval_logps/rejected": -7.506450176239014,
      "eval_loss": 0.21870465576648712,
      "eval_rewards/accuracies": 0.9801980257034302,
      "eval_rewards/chosen": -5.171125888824463,
      "eval_rewards/margins": 6.088548183441162,
      "eval_rewards/rejected": -11.259674072265625,
      "eval_runtime": 29.1008,
      "eval_samples_per_second": 27.525,
      "eval_steps_per_second": 3.471,
      "step": 900
    },
    {
      "epoch": 0.7346115035317861,
      "grad_norm": 3.623443841934204,
      "learning_rate": 1.677833383153542e-06,
      "logits/chosen": 2.915825605392456,
      "logits/rejected": 2.7418575286865234,
      "logps/chosen": -3.517927646636963,
      "logps/rejected": -8.27934455871582,
      "loss": 0.2311,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.276891708374023,
      "rewards/margins": 7.142125129699707,
      "rewards/rejected": -12.419015884399414,
      "step": 910
    },
    {
      "epoch": 0.7426841574167508,
      "grad_norm": 4.020949363708496,
      "learning_rate": 1.6285698816954626e-06,
      "logits/chosen": 2.4125845432281494,
      "logits/rejected": 2.6929471492767334,
      "logps/chosen": -3.646517276763916,
      "logps/rejected": -7.9424262046813965,
      "loss": 0.2034,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.469775199890137,
      "rewards/margins": 6.443863868713379,
      "rewards/rejected": -11.913639068603516,
      "step": 920
    },
    {
      "epoch": 0.7507568113017155,
      "grad_norm": 4.215940475463867,
      "learning_rate": 1.5796886182883053e-06,
      "logits/chosen": 2.525031566619873,
      "logits/rejected": 2.478022336959839,
      "logps/chosen": -3.580151319503784,
      "logps/rejected": -8.356379508972168,
      "loss": 0.2301,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.370226860046387,
      "rewards/margins": 7.164343357086182,
      "rewards/rejected": -12.534570693969727,
      "step": 930
    },
    {
      "epoch": 0.7588294651866802,
      "grad_norm": 3.3429832458496094,
      "learning_rate": 1.5312110338697427e-06,
      "logits/chosen": 2.68546462059021,
      "logits/rejected": 2.4535439014434814,
      "logps/chosen": -3.716118574142456,
      "logps/rejected": -8.706206321716309,
      "loss": 0.1898,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5741777420043945,
      "rewards/margins": 7.485130310058594,
      "rewards/rejected": -13.059308052062988,
      "step": 940
    },
    {
      "epoch": 0.7669021190716448,
      "grad_norm": 3.5254878997802734,
      "learning_rate": 1.4831583923105e-06,
      "logits/chosen": 2.924198627471924,
      "logits/rejected": 2.7865915298461914,
      "logps/chosen": -3.7330048084259033,
      "logps/rejected": -8.79862117767334,
      "loss": 0.1947,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5995073318481445,
      "rewards/margins": 7.598424434661865,
      "rewards/rejected": -13.197932243347168,
      "step": 950
    },
    {
      "epoch": 0.7669021190716448,
      "eval_logits/chosen": 2.5911405086517334,
      "eval_logits/rejected": 2.4414517879486084,
      "eval_logps/chosen": -3.396683692932129,
      "eval_logps/rejected": -7.610343933105469,
      "eval_loss": 0.2162380963563919,
      "eval_rewards/accuracies": 0.9900990128517151,
      "eval_rewards/chosen": -5.09502649307251,
      "eval_rewards/margins": 6.32049036026001,
      "eval_rewards/rejected": -11.415514945983887,
      "eval_runtime": 29.0842,
      "eval_samples_per_second": 27.541,
      "eval_steps_per_second": 3.473,
      "step": 950
    },
    {
      "epoch": 0.7749747729566094,
      "grad_norm": 2.4495151042938232,
      "learning_rate": 1.4355517710873184e-06,
      "logits/chosen": 3.756882429122925,
      "logits/rejected": 3.461829423904419,
      "logps/chosen": -3.4112212657928467,
      "logps/rejected": -7.716673374176025,
      "loss": 0.1954,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.1168317794799805,
      "rewards/margins": 6.458177089691162,
      "rewards/rejected": -11.575010299682617,
      "step": 960
    },
    {
      "epoch": 0.7830474268415741,
      "grad_norm": 3.6960482597351074,
      "learning_rate": 1.388412052037682e-06,
      "logits/chosen": 2.683379888534546,
      "logits/rejected": 2.5084214210510254,
      "logps/chosen": -3.2802653312683105,
      "logps/rejected": -8.223787307739258,
      "loss": 0.1962,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.920398712158203,
      "rewards/margins": 7.415283203125,
      "rewards/rejected": -12.33568000793457,
      "step": 970
    },
    {
      "epoch": 0.7911200807265388,
      "grad_norm": 1.6209157705307007,
      "learning_rate": 1.3417599122003464e-06,
      "logits/chosen": 2.2775497436523438,
      "logits/rejected": 2.4255974292755127,
      "logps/chosen": -3.296114444732666,
      "logps/rejected": -7.983359336853027,
      "loss": 0.1651,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.944170951843262,
      "rewards/margins": 7.030867576599121,
      "rewards/rejected": -11.975038528442383,
      "step": 980
    },
    {
      "epoch": 0.7991927346115035,
      "grad_norm": 4.007768154144287,
      "learning_rate": 1.2956158147457116e-06,
      "logits/chosen": 3.2154605388641357,
      "logits/rejected": 2.895961046218872,
      "logps/chosen": -3.3481571674346924,
      "logps/rejected": -8.350494384765625,
      "loss": 0.1702,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.022235870361328,
      "rewards/margins": 7.503505706787109,
      "rewards/rejected": -12.525741577148438,
      "step": 990
    },
    {
      "epoch": 0.8072653884964682,
      "grad_norm": 3.026970386505127,
      "learning_rate": 1.2500000000000007e-06,
      "logits/chosen": 3.6043121814727783,
      "logits/rejected": 3.541698932647705,
      "logps/chosen": -3.722698926925659,
      "logps/rejected": -8.218320846557617,
      "loss": 0.1934,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.584048271179199,
      "rewards/margins": 6.743433475494385,
      "rewards/rejected": -12.327482223510742,
      "step": 1000
    },
    {
      "epoch": 0.8072653884964682,
      "eval_logits/chosen": 2.707226514816284,
      "eval_logits/rejected": 2.5214099884033203,
      "eval_logps/chosen": -3.502831220626831,
      "eval_logps/rejected": -7.705867767333984,
      "eval_loss": 0.20982278883457184,
      "eval_rewards/accuracies": 0.9900990128517151,
      "eval_rewards/chosen": -5.254247188568115,
      "eval_rewards/margins": 6.304553985595703,
      "eval_rewards/rejected": -11.558801651000977,
      "eval_runtime": 29.0974,
      "eval_samples_per_second": 27.528,
      "eval_steps_per_second": 3.471,
      "step": 1000
    },
    {
      "epoch": 0.8153380423814329,
      "grad_norm": 2.1142029762268066,
      "learning_rate": 1.204932476567175e-06,
      "logits/chosen": 3.21277117729187,
      "logits/rejected": 2.902198314666748,
      "logps/chosen": -3.9205710887908936,
      "logps/rejected": -8.282219886779785,
      "loss": 0.1911,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.880856513977051,
      "rewards/margins": 6.542473793029785,
      "rewards/rejected": -12.423330307006836,
      "step": 1010
    },
    {
      "epoch": 0.8234106962663976,
      "grad_norm": 4.392597198486328,
      "learning_rate": 1.160433012552508e-06,
      "logits/chosen": 2.785994052886963,
      "logits/rejected": 2.8172032833099365,
      "logps/chosen": -3.6294891834259033,
      "logps/rejected": -8.542880058288574,
      "loss": 0.2222,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.4442338943481445,
      "rewards/margins": 7.370086669921875,
      "rewards/rejected": -12.81432056427002,
      "step": 1020
    },
    {
      "epoch": 0.8314833501513622,
      "grad_norm": 2.633239269256592,
      "learning_rate": 1.11652112689164e-06,
      "logits/chosen": 2.663726329803467,
      "logits/rejected": 2.4244682788848877,
      "logps/chosen": -3.625394821166992,
      "logps/rejected": -8.94005298614502,
      "loss": 0.1768,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.438092231750488,
      "rewards/margins": 7.971987247467041,
      "rewards/rejected": -13.410079956054688,
      "step": 1030
    },
    {
      "epoch": 0.8395560040363269,
      "grad_norm": 2.5595297813415527,
      "learning_rate": 1.073216080788921e-06,
      "logits/chosen": 3.432141065597534,
      "logits/rejected": 3.068664789199829,
      "logps/chosen": -3.4811911582946777,
      "logps/rejected": -8.416011810302734,
      "loss": 0.1923,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.221786975860596,
      "rewards/margins": 7.402231693267822,
      "rewards/rejected": -12.624017715454102,
      "step": 1040
    },
    {
      "epoch": 0.8476286579212916,
      "grad_norm": 5.044093132019043,
      "learning_rate": 1.0305368692688175e-06,
      "logits/chosen": 2.554561138153076,
      "logits/rejected": 2.273308753967285,
      "logps/chosen": -3.726578950881958,
      "logps/rejected": -8.446795463562012,
      "loss": 0.1525,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.589868545532227,
      "rewards/margins": 7.080326080322266,
      "rewards/rejected": -12.670194625854492,
      "step": 1050
    },
    {
      "epoch": 0.8476286579212916,
      "eval_logits/chosen": 2.738980770111084,
      "eval_logits/rejected": 2.524784564971924,
      "eval_logps/chosen": -3.4412803649902344,
      "eval_logps/rejected": -7.715125560760498,
      "eval_loss": 0.20868732035160065,
      "eval_rewards/accuracies": 0.9801980257034302,
      "eval_rewards/chosen": -5.161920070648193,
      "eval_rewards/margins": 6.410768032073975,
      "eval_rewards/rejected": -11.572687149047852,
      "eval_runtime": 29.0996,
      "eval_samples_per_second": 27.526,
      "eval_steps_per_second": 3.471,
      "step": 1050
    },
    {
      "epoch": 0.8557013118062563,
      "grad_norm": 4.388390064239502,
      "learning_rate": 9.88502212844063e-07,
      "logits/chosen": 3.056361198425293,
      "logits/rejected": 2.8604633808135986,
      "logps/chosen": -3.4161269664764404,
      "logps/rejected": -8.637321472167969,
      "loss": 0.1962,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.124190807342529,
      "rewards/margins": 7.831790924072266,
      "rewards/rejected": -12.955981254577637,
      "step": 1060
    },
    {
      "epoch": 0.863773965691221,
      "grad_norm": 5.200846195220947,
      "learning_rate": 9.471305493042243e-07,
      "logits/chosen": 2.426769971847534,
      "logits/rejected": 2.714139461517334,
      "logps/chosen": -3.3948585987091064,
      "logps/rejected": -8.400737762451172,
      "loss": 0.1953,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.092287540435791,
      "rewards/margins": 7.508819580078125,
      "rewards/rejected": -12.601107597351074,
      "step": 1070
    },
    {
      "epoch": 0.8718466195761857,
      "grad_norm": 4.115231513977051,
      "learning_rate": 9.064400256282757e-07,
      "logits/chosen": 2.680354356765747,
      "logits/rejected": 2.4186928272247314,
      "logps/chosen": -3.406576633453369,
      "logps/rejected": -8.102071762084961,
      "loss": 0.1961,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.109864711761475,
      "rewards/margins": 7.04324197769165,
      "rewards/rejected": -12.153106689453125,
      "step": 1080
    },
    {
      "epoch": 0.8799192734611504,
      "grad_norm": 2.0617411136627197,
      "learning_rate": 8.664484900247363e-07,
      "logits/chosen": 2.8944904804229736,
      "logits/rejected": 2.763841152191162,
      "logps/chosen": -3.420555591583252,
      "logps/rejected": -8.535276412963867,
      "loss": 0.2107,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.130833625793457,
      "rewards/margins": 7.672080993652344,
      "rewards/rejected": -12.802912712097168,
      "step": 1090
    },
    {
      "epoch": 0.887991927346115,
      "grad_norm": 4.412430763244629,
      "learning_rate": 8.271734841028553e-07,
      "logits/chosen": 2.8057053089141846,
      "logits/rejected": 2.880598545074463,
      "logps/chosen": -3.727388381958008,
      "logps/rejected": -8.665179252624512,
      "loss": 0.2103,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.59108304977417,
      "rewards/margins": 7.406687259674072,
      "rewards/rejected": -12.997769355773926,
      "step": 1100
    },
    {
      "epoch": 0.887991927346115,
      "eval_logits/chosen": 2.784738063812256,
      "eval_logits/rejected": 2.529378890991211,
      "eval_logps/chosen": -3.4977424144744873,
      "eval_logps/rejected": -7.756742477416992,
      "eval_loss": 0.2044122815132141,
      "eval_rewards/accuracies": 0.9801980257034302,
      "eval_rewards/chosen": -5.2466139793396,
      "eval_rewards/margins": 6.388499736785889,
      "eval_rewards/rejected": -11.635113716125488,
      "eval_runtime": 29.0833,
      "eval_samples_per_second": 27.542,
      "eval_steps_per_second": 3.473,
      "step": 1100
    },
    {
      "epoch": 0.8960645812310797,
      "grad_norm": 3.327115058898926,
      "learning_rate": 7.886322351782782e-07,
      "logits/chosen": 1.8973369598388672,
      "logits/rejected": 1.9729961156845093,
      "logps/chosen": -3.3052139282226562,
      "logps/rejected": -8.953332901000977,
      "loss": 0.2092,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.957820892333984,
      "rewards/margins": 8.472179412841797,
      "rewards/rejected": -13.430000305175781,
      "step": 1110
    },
    {
      "epoch": 0.9041372351160444,
      "grad_norm": 5.0973358154296875,
      "learning_rate": 7.508416487165862e-07,
      "logits/chosen": 2.4597842693328857,
      "logits/rejected": 2.5812220573425293,
      "logps/chosen": -3.381796360015869,
      "logps/rejected": -8.363035202026367,
      "loss": 0.2077,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.072694301605225,
      "rewards/margins": 7.471858978271484,
      "rewards/rejected": -12.544553756713867,
      "step": 1120
    },
    {
      "epoch": 0.9122098890010091,
      "grad_norm": 5.166894912719727,
      "learning_rate": 7.138183009179922e-07,
      "logits/chosen": 3.5492165088653564,
      "logits/rejected": 3.2513375282287598,
      "logps/chosen": -3.6195359230041504,
      "logps/rejected": -8.38493824005127,
      "loss": 0.1904,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.4293036460876465,
      "rewards/margins": 7.1481032371521,
      "rewards/rejected": -12.577406883239746,
      "step": 1130
    },
    {
      "epoch": 0.9202825428859738,
      "grad_norm": 3.683342695236206,
      "learning_rate": 6.775784314464717e-07,
      "logits/chosen": 2.7155990600585938,
      "logits/rejected": 2.7459475994110107,
      "logps/chosen": -4.0417985916137695,
      "logps/rejected": -9.647562026977539,
      "loss": 0.2156,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.062697410583496,
      "rewards/margins": 8.40864372253418,
      "rewards/rejected": -14.471341133117676,
      "step": 1140
    },
    {
      "epoch": 0.9283551967709385,
      "grad_norm": 4.6415815353393555,
      "learning_rate": 6.421379363065142e-07,
      "logits/chosen": 2.0692670345306396,
      "logits/rejected": 2.390552043914795,
      "logps/chosen": -3.8512816429138184,
      "logps/rejected": -8.84770393371582,
      "loss": 0.2165,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.776922702789307,
      "rewards/margins": 7.494633674621582,
      "rewards/rejected": -13.271554946899414,
      "step": 1150
    },
    {
      "epoch": 0.9283551967709385,
      "eval_logits/chosen": 2.716364860534668,
      "eval_logits/rejected": 2.517240047454834,
      "eval_logps/chosen": -3.552220344543457,
      "eval_logps/rejected": -7.850137710571289,
      "eval_loss": 0.19964782893657684,
      "eval_rewards/accuracies": 0.9801980257034302,
      "eval_rewards/chosen": -5.328330993652344,
      "eval_rewards/margins": 6.446875095367432,
      "eval_rewards/rejected": -11.775205612182617,
      "eval_runtime": 29.0905,
      "eval_samples_per_second": 27.535,
      "eval_steps_per_second": 3.472,
      "step": 1150
    },
    {
      "epoch": 0.9364278506559032,
      "grad_norm": 3.169071912765503,
      "learning_rate": 6.075123608706093e-07,
      "logits/chosen": 3.026804208755493,
      "logits/rejected": 2.92323637008667,
      "logps/chosen": -3.5037333965301514,
      "logps/rejected": -8.634604454040527,
      "loss": 0.1896,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.255600929260254,
      "rewards/margins": 7.696307182312012,
      "rewards/rejected": -12.95190715789795,
      "step": 1160
    },
    {
      "epoch": 0.9445005045408678,
      "grad_norm": 3.537233352661133,
      "learning_rate": 5.737168930605272e-07,
      "logits/chosen": 1.8126366138458252,
      "logits/rejected": 2.0001204013824463,
      "logps/chosen": -3.8311641216278076,
      "logps/rejected": -8.98426342010498,
      "loss": 0.195,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.74674654006958,
      "rewards/margins": 7.729648590087891,
      "rewards/rejected": -13.476394653320312,
      "step": 1170
    },
    {
      "epoch": 0.9525731584258325,
      "grad_norm": 4.516322612762451,
      "learning_rate": 5.407663566854008e-07,
      "logits/chosen": 2.7332987785339355,
      "logits/rejected": 2.6524689197540283,
      "logps/chosen": -3.8542847633361816,
      "logps/rejected": -8.629236221313477,
      "loss": 0.171,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.781428337097168,
      "rewards/margins": 7.162425994873047,
      "rewards/rejected": -12.943852424621582,
      "step": 1180
    },
    {
      "epoch": 0.9606458123107972,
      "grad_norm": 3.578998327255249,
      "learning_rate": 5.086752049395094e-07,
      "logits/chosen": 3.0999205112457275,
      "logits/rejected": 3.115819215774536,
      "logps/chosen": -3.870943784713745,
      "logps/rejected": -8.126202583312988,
      "loss": 0.2047,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.8064165115356445,
      "rewards/margins": 6.382887363433838,
      "rewards/rejected": -12.189303398132324,
      "step": 1190
    },
    {
      "epoch": 0.9687184661957619,
      "grad_norm": 4.2472405433654785,
      "learning_rate": 4.774575140626317e-07,
      "logits/chosen": 2.8590927124023438,
      "logits/rejected": 2.744704484939575,
      "logps/chosen": -4.1155290603637695,
      "logps/rejected": -8.967794418334961,
      "loss": 0.2222,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.173293590545654,
      "rewards/margins": 7.2783966064453125,
      "rewards/rejected": -13.451690673828125,
      "step": 1200
    },
    {
      "epoch": 0.9687184661957619,
      "eval_logits/chosen": 2.7425291538238525,
      "eval_logits/rejected": 2.552403211593628,
      "eval_logps/chosen": -3.515270948410034,
      "eval_logps/rejected": -7.858344554901123,
      "eval_loss": 0.2018371820449829,
      "eval_rewards/accuracies": 0.9900990128517151,
      "eval_rewards/chosen": -5.27290678024292,
      "eval_rewards/margins": 6.514610290527344,
      "eval_rewards/rejected": -11.787515640258789,
      "eval_runtime": 29.1438,
      "eval_samples_per_second": 27.484,
      "eval_steps_per_second": 3.466,
      "step": 1200
    },
    {
      "epoch": 0.9767911200807265,
      "grad_norm": 3.373248815536499,
      "learning_rate": 4.4712697716573994e-07,
      "logits/chosen": 2.8510289192199707,
      "logits/rejected": 2.680002212524414,
      "logps/chosen": -3.8011245727539062,
      "logps/rejected": -9.465116500854492,
      "loss": 0.18,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.701686859130859,
      "rewards/margins": 8.495986938476562,
      "rewards/rejected": -14.197672843933105,
      "step": 1210
    },
    {
      "epoch": 0.9848637739656912,
      "grad_norm": 4.309781551361084,
      "learning_rate": 4.1769689822475147e-07,
      "logits/chosen": 3.285893678665161,
      "logits/rejected": 2.8214175701141357,
      "logps/chosen": -3.687873363494873,
      "logps/rejected": -8.890458106994629,
      "loss": 0.1536,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.5318098068237305,
      "rewards/margins": 7.803877353668213,
      "rewards/rejected": -13.335685729980469,
      "step": 1220
    },
    {
      "epoch": 0.992936427850656,
      "grad_norm": 4.0880208015441895,
      "learning_rate": 3.891801862449629e-07,
      "logits/chosen": 2.9707233905792236,
      "logits/rejected": 3.113229274749756,
      "logps/chosen": -3.793541669845581,
      "logps/rejected": -8.105633735656738,
      "loss": 0.1782,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.69031286239624,
      "rewards/margins": 6.468136787414551,
      "rewards/rejected": -12.15845012664795,
      "step": 1230
    },
    {
      "epoch": 1.001614530776993,
      "grad_norm": 4.56434965133667,
      "learning_rate": 3.615893495987335e-07,
      "logits/chosen": 1.8961296081542969,
      "logits/rejected": 1.7723509073257446,
      "logps/chosen": -3.2175498008728027,
      "logps/rejected": -8.063615798950195,
      "loss": 0.2248,
      "rewards/accuracies": 0.929411768913269,
      "rewards/chosen": -4.826324939727783,
      "rewards/margins": 7.26909875869751,
      "rewards/rejected": -12.095422744750977,
      "step": 1240
    },
    {
      "epoch": 1.0096871846619577,
      "grad_norm": 3.1112208366394043,
      "learning_rate": 3.3493649053890325e-07,
      "logits/chosen": 2.197903871536255,
      "logits/rejected": 2.412510395050049,
      "logps/chosen": -3.3189709186553955,
      "logps/rejected": -8.700540542602539,
      "loss": 0.1499,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.978456974029541,
      "rewards/margins": 8.072355270385742,
      "rewards/rejected": -13.050811767578125,
      "step": 1250
    },
    {
      "epoch": 1.0096871846619577,
      "eval_logits/chosen": 2.780805826187134,
      "eval_logits/rejected": 2.5666725635528564,
      "eval_logps/chosen": -3.482585906982422,
      "eval_logps/rejected": -7.832629680633545,
      "eval_loss": 0.1983107328414917,
      "eval_rewards/accuracies": 0.9801980257034302,
      "eval_rewards/chosen": -5.223879337310791,
      "eval_rewards/margins": 6.525064468383789,
      "eval_rewards/rejected": -11.748945236206055,
      "eval_runtime": 29.0849,
      "eval_samples_per_second": 27.54,
      "eval_steps_per_second": 3.473,
      "step": 1250
    },
    {
      "epoch": 1.0177598385469222,
      "grad_norm": 5.360677242279053,
      "learning_rate": 3.092332998903416e-07,
      "logits/chosen": 3.3109779357910156,
      "logits/rejected": 3.268786907196045,
      "logps/chosen": -3.612884998321533,
      "logps/rejected": -8.643686294555664,
      "loss": 0.1847,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.419327735900879,
      "rewards/margins": 7.546202182769775,
      "rewards/rejected": -12.965530395507812,
      "step": 1260
    },
    {
      "epoch": 1.025832492431887,
      "grad_norm": 2.412815809249878,
      "learning_rate": 2.844910519219632e-07,
      "logits/chosen": 2.8832778930664062,
      "logits/rejected": 2.6380584239959717,
      "logps/chosen": -3.5998082160949707,
      "logps/rejected": -8.734532356262207,
      "loss": 0.1894,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.399710655212402,
      "rewards/margins": 7.70208740234375,
      "rewards/rejected": -13.101797103881836,
      "step": 1270
    },
    {
      "epoch": 1.0339051463168516,
      "grad_norm": 3.3461902141571045,
      "learning_rate": 2.6072059940146775e-07,
      "logits/chosen": 2.6302340030670166,
      "logits/rejected": 2.664712905883789,
      "logps/chosen": -3.3908112049102783,
      "logps/rejected": -8.409881591796875,
      "loss": 0.1791,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.086216926574707,
      "rewards/margins": 7.528606414794922,
      "rewards/rejected": -12.614823341369629,
      "step": 1280
    },
    {
      "epoch": 1.0419778002018163,
      "grad_norm": 3.360499620437622,
      "learning_rate": 2.3793236883495164e-07,
      "logits/chosen": 3.280123233795166,
      "logits/rejected": 3.131016254425049,
      "logps/chosen": -3.890662670135498,
      "logps/rejected": -8.402752876281738,
      "loss": 0.21,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.835994243621826,
      "rewards/margins": 6.768135070800781,
      "rewards/rejected": -12.60412883758545,
      "step": 1290
    },
    {
      "epoch": 1.050050454086781,
      "grad_norm": 3.2999982833862305,
      "learning_rate": 2.1613635589349756e-07,
      "logits/chosen": 3.0537660121917725,
      "logits/rejected": 2.9963178634643555,
      "logps/chosen": -3.6433207988739014,
      "logps/rejected": -9.105337142944336,
      "loss": 0.1632,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.464980602264404,
      "rewards/margins": 8.193026542663574,
      "rewards/rejected": -13.658007621765137,
      "step": 1300
    },
    {
      "epoch": 1.050050454086781,
      "eval_logits/chosen": 2.8208885192871094,
      "eval_logits/rejected": 2.606031656265259,
      "eval_logps/chosen": -3.4751346111297607,
      "eval_logps/rejected": -7.840829849243164,
      "eval_loss": 0.1993524730205536,
      "eval_rewards/accuracies": 0.9801980257034302,
      "eval_rewards/chosen": -5.212701320648193,
      "eval_rewards/margins": 6.548543930053711,
      "eval_rewards/rejected": -11.761244773864746,
      "eval_runtime": 29.2282,
      "eval_samples_per_second": 27.405,
      "eval_steps_per_second": 3.456,
      "step": 1300
    }
  ],
  "logging_steps": 10,
  "max_steps": 1500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 3.027748725154906e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}