checkpoint-750/trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.6054490413723511,
  "eval_steps": 50,
  "global_step": 750,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.008072653884964682,
      "grad_norm": 0.04926518723368645,
      "learning_rate": 4.999451708687114e-06,
      "logits/chosen": 14.542106628417969,
      "logits/rejected": 14.864250183105469,
      "logps/chosen": -0.2809702754020691,
      "logps/rejected": -0.3013763725757599,
      "loss": 0.9221,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.421455442905426,
      "rewards/margins": 0.03060910664498806,
      "rewards/rejected": -0.452064573764801,
      "step": 10
    },
    {
      "epoch": 0.016145307769929364,
      "grad_norm": 0.056249432265758514,
      "learning_rate": 4.997807075247147e-06,
      "logits/chosen": 14.614748001098633,
      "logits/rejected": 15.259109497070312,
      "logps/chosen": -0.2828002870082855,
      "logps/rejected": -0.3477819561958313,
      "loss": 0.916,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.4242004454135895,
      "rewards/margins": 0.09747247397899628,
      "rewards/rejected": -0.5216729044914246,
      "step": 20
    },
    {
      "epoch": 0.024217961654894045,
      "grad_norm": 0.0664869099855423,
      "learning_rate": 4.9950668210706795e-06,
      "logits/chosen": 14.391204833984375,
      "logits/rejected": 14.82734203338623,
      "logps/chosen": -0.287629634141922,
      "logps/rejected": -0.3329126834869385,
      "loss": 0.914,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.4314444661140442,
      "rewards/margins": 0.06792456656694412,
      "rewards/rejected": -0.4993689954280853,
      "step": 30
    },
    {
      "epoch": 0.03229061553985873,
      "grad_norm": 0.055584829300642014,
      "learning_rate": 4.9912321481237616e-06,
      "logits/chosen": 13.93278980255127,
      "logits/rejected": 14.886846542358398,
      "logps/chosen": -0.28155946731567383,
      "logps/rejected": -0.3678051829338074,
      "loss": 0.9273,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.42233920097351074,
      "rewards/margins": 0.12936851382255554,
      "rewards/rejected": -0.5517078042030334,
      "step": 40
    },
    {
      "epoch": 0.04036326942482341,
      "grad_norm": 0.07997103035449982,
      "learning_rate": 4.986304738420684e-06,
      "logits/chosen": 14.172693252563477,
      "logits/rejected": 14.742494583129883,
      "logps/chosen": -0.28225988149642944,
      "logps/rejected": -0.34329456090927124,
      "loss": 0.9081,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.42338982224464417,
      "rewards/margins": 0.0915520042181015,
      "rewards/rejected": -0.5149418115615845,
      "step": 50
    },
    {
      "epoch": 0.04036326942482341,
      "eval_logits/chosen": 14.200166702270508,
      "eval_logits/rejected": 14.817726135253906,
      "eval_logps/chosen": -0.2625390887260437,
      "eval_logps/rejected": -0.3458769917488098,
      "eval_loss": 0.9080610275268555,
      "eval_rewards/accuracies": 0.5544554591178894,
      "eval_rewards/chosen": -0.39380866289138794,
      "eval_rewards/margins": 0.1250067949295044,
      "eval_rewards/rejected": -0.5188154578208923,
      "eval_runtime": 29.8098,
      "eval_samples_per_second": 26.87,
      "eval_steps_per_second": 3.388,
      "step": 50
    },
    {
      "epoch": 0.04843592330978809,
      "grad_norm": 0.06322095543146133,
      "learning_rate": 4.980286753286196e-06,
      "logits/chosen": 14.008010864257812,
      "logits/rejected": 14.975939750671387,
      "logps/chosen": -0.2697351574897766,
      "logps/rejected": -0.3445274233818054,
      "loss": 0.9195,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.40460270643234253,
      "rewards/margins": 0.1121884360909462,
      "rewards/rejected": -0.5167912244796753,
      "step": 60
    },
    {
      "epoch": 0.056508577194752774,
      "grad_norm": 0.12975476682186127,
      "learning_rate": 4.973180832407471e-06,
      "logits/chosen": 14.213134765625,
      "logits/rejected": 14.889978408813477,
      "logps/chosen": -0.2839818596839905,
      "logps/rejected": -0.3382417559623718,
      "loss": 0.9241,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.4259727895259857,
      "rewards/margins": 0.08138985931873322,
      "rewards/rejected": -0.5073626637458801,
      "step": 70
    },
    {
      "epoch": 0.06458123107971746,
      "grad_norm": 0.08352109789848328,
      "learning_rate": 4.964990092676263e-06,
      "logits/chosen": 14.395146369934082,
      "logits/rejected": 14.900177001953125,
      "logps/chosen": -0.27067264914512634,
      "logps/rejected": -0.3516673743724823,
      "loss": 0.9251,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.4060089588165283,
      "rewards/margins": 0.12149210274219513,
      "rewards/rejected": -0.5275009870529175,
      "step": 80
    },
    {
      "epoch": 0.07265388496468214,
      "grad_norm": 0.0793827548623085,
      "learning_rate": 4.9557181268217225e-06,
      "logits/chosen": 13.905145645141602,
      "logits/rejected": 14.897878646850586,
      "logps/chosen": -0.26814645528793335,
      "logps/rejected": -0.38061466813087463,
      "loss": 0.9055,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4022197127342224,
      "rewards/margins": 0.16870227456092834,
      "rewards/rejected": -0.5709219574928284,
      "step": 90
    },
    {
      "epoch": 0.08072653884964683,
      "grad_norm": 0.09212008118629456,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": 14.11627197265625,
      "logits/rejected": 14.571977615356445,
      "logps/chosen": -0.3130624294281006,
      "logps/rejected": -0.35073375701904297,
      "loss": 0.911,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.46959367394447327,
      "rewards/margins": 0.05650699883699417,
      "rewards/rejected": -0.5261006951332092,
      "step": 100
    },
    {
      "epoch": 0.08072653884964683,
      "eval_logits/chosen": 13.845876693725586,
      "eval_logits/rejected": 14.490789413452148,
      "eval_logps/chosen": -0.2531408667564392,
      "eval_logps/rejected": -0.3464036285877228,
      "eval_loss": 0.8986235857009888,
      "eval_rewards/accuracies": 0.5544554591178894,
      "eval_rewards/chosen": -0.3797112703323364,
      "eval_rewards/margins": 0.13989417254924774,
      "eval_rewards/rejected": -0.5196054577827454,
      "eval_runtime": 29.0914,
      "eval_samples_per_second": 27.534,
      "eval_steps_per_second": 3.472,
      "step": 100
    },
    {
      "epoch": 0.08879919273461151,
      "grad_norm": 0.08554862439632416,
      "learning_rate": 4.933947257182901e-06,
      "logits/chosen": 13.416229248046875,
      "logits/rejected": 14.582674026489258,
      "logps/chosen": -0.2523443102836609,
      "logps/rejected": -0.38751405477523804,
      "loss": 0.8993,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3785164952278137,
      "rewards/margins": 0.20275457203388214,
      "rewards/rejected": -0.5812710523605347,
      "step": 110
    },
    {
      "epoch": 0.09687184661957618,
      "grad_norm": 0.29209578037261963,
      "learning_rate": 4.921457902821578e-06,
      "logits/chosen": 13.874654769897461,
      "logits/rejected": 14.423624992370605,
      "logps/chosen": -0.27300480008125305,
      "logps/rejected": -0.3526575267314911,
      "loss": 0.8942,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.40950721502304077,
      "rewards/margins": 0.11947910487651825,
      "rewards/rejected": -0.5289863348007202,
      "step": 120
    },
    {
      "epoch": 0.10494450050454086,
      "grad_norm": 0.1028478816151619,
      "learning_rate": 4.907906416994146e-06,
      "logits/chosen": 14.030723571777344,
      "logits/rejected": 14.711235046386719,
      "logps/chosen": -0.27410784363746643,
      "logps/rejected": -0.3665519058704376,
      "loss": 0.8922,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.41116175055503845,
      "rewards/margins": 0.1386660784482956,
      "rewards/rejected": -0.5498278737068176,
      "step": 130
    },
    {
      "epoch": 0.11301715438950555,
      "grad_norm": 0.08459590375423431,
      "learning_rate": 4.893298743830168e-06,
      "logits/chosen": 13.477182388305664,
      "logits/rejected": 13.919464111328125,
      "logps/chosen": -0.2495063841342926,
      "logps/rejected": -0.3594816029071808,
      "loss": 0.8957,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.3742595613002777,
      "rewards/margins": 0.16496284306049347,
      "rewards/rejected": -0.5392224192619324,
      "step": 140
    },
    {
      "epoch": 0.12108980827447023,
      "grad_norm": 0.12467797845602036,
      "learning_rate": 4.8776412907378845e-06,
      "logits/chosen": 12.801656723022461,
      "logits/rejected": 13.564155578613281,
      "logps/chosen": -0.2779986262321472,
      "logps/rejected": -0.33566445112228394,
      "loss": 0.892,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4169979691505432,
      "rewards/margins": 0.08649872243404388,
      "rewards/rejected": -0.5034967064857483,
      "step": 150
    },
    {
      "epoch": 0.12108980827447023,
      "eval_logits/chosen": 12.169118881225586,
      "eval_logits/rejected": 13.019683837890625,
      "eval_logps/chosen": -0.25547870993614197,
      "eval_logps/rejected": -0.37277930974960327,
      "eval_loss": 0.8789658546447754,
      "eval_rewards/accuracies": 0.5742574334144592,
      "eval_rewards/chosen": -0.38321802020072937,
      "eval_rewards/margins": 0.17595094442367554,
      "eval_rewards/rejected": -0.5591689944267273,
      "eval_runtime": 29.0961,
      "eval_samples_per_second": 27.529,
      "eval_steps_per_second": 3.471,
      "step": 150
    },
    {
      "epoch": 0.12916246215943492,
      "grad_norm": 0.1712370663881302,
      "learning_rate": 4.860940925593703e-06,
      "logits/chosen": 11.972528457641602,
      "logits/rejected": 12.684088706970215,
      "logps/chosen": -0.28340521454811096,
      "logps/rejected": -0.3864063024520874,
      "loss": 0.8789,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.42510780692100525,
      "rewards/margins": 0.15450166165828705,
      "rewards/rejected": -0.5796095132827759,
      "step": 160
    },
    {
      "epoch": 0.13723511604439959,
      "grad_norm": 0.21750673651695251,
      "learning_rate": 4.84320497372973e-06,
      "logits/chosen": 11.93881607055664,
      "logits/rejected": 12.458230972290039,
      "logps/chosen": -0.274599552154541,
      "logps/rejected": -0.3882916271686554,
      "loss": 0.8857,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4118993282318115,
      "rewards/margins": 0.1705380380153656,
      "rewards/rejected": -0.5824374556541443,
      "step": 170
    },
    {
      "epoch": 0.14530776992936428,
      "grad_norm": 0.16000741720199585,
      "learning_rate": 4.824441214720629e-06,
      "logits/chosen": 9.631464958190918,
      "logits/rejected": 10.772969245910645,
      "logps/chosen": -0.287865549325943,
      "logps/rejected": -0.4817379415035248,
      "loss": 0.8699,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.4317983090877533,
      "rewards/margins": 0.29080861806869507,
      "rewards/rejected": -0.722606897354126,
      "step": 180
    },
    {
      "epoch": 0.15338042381432895,
      "grad_norm": 0.46293890476226807,
      "learning_rate": 4.804657878971252e-06,
      "logits/chosen": 9.081937789916992,
      "logits/rejected": 10.024572372436523,
      "logps/chosen": -0.2941994071006775,
      "logps/rejected": -0.4772109389305115,
      "loss": 0.8565,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.44129911065101624,
      "rewards/margins": 0.2745172679424286,
      "rewards/rejected": -0.7158163785934448,
      "step": 190
    },
    {
      "epoch": 0.16145307769929365,
      "grad_norm": 0.24658434092998505,
      "learning_rate": 4.783863644106502e-06,
      "logits/chosen": 7.767125606536865,
      "logits/rejected": 8.009145736694336,
      "logps/chosen": -0.32877305150032043,
      "logps/rejected": -0.47733697295188904,
      "loss": 0.8482,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.49315959215164185,
      "rewards/margins": 0.2228458821773529,
      "rewards/rejected": -0.7160054445266724,
      "step": 200
    },
    {
      "epoch": 0.16145307769929365,
      "eval_logits/chosen": 6.716187953948975,
      "eval_logits/rejected": 7.248146057128906,
      "eval_logps/chosen": -0.30652713775634766,
      "eval_logps/rejected": -0.5056277513504028,
      "eval_loss": 0.8243693113327026,
      "eval_rewards/accuracies": 0.603960394859314,
      "eval_rewards/chosen": -0.4597907066345215,
      "eval_rewards/margins": 0.2986510097980499,
      "eval_rewards/rejected": -0.7584417462348938,
      "eval_runtime": 29.0989,
      "eval_samples_per_second": 27.527,
      "eval_steps_per_second": 3.471,
      "step": 200
    },
    {
      "epoch": 0.16952573158425832,
      "grad_norm": 0.40796443819999695,
      "learning_rate": 4.762067631165049e-06,
      "logits/chosen": 5.932644844055176,
      "logits/rejected": 6.521953582763672,
      "logps/chosen": -0.32976508140563965,
      "logps/rejected": -0.5628186464309692,
      "loss": 0.7988,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.4946475923061371,
      "rewards/margins": 0.3495803475379944,
      "rewards/rejected": -0.8442279696464539,
      "step": 210
    },
    {
      "epoch": 0.17759838546922302,
      "grad_norm": 0.42468318343162537,
      "learning_rate": 4.7392794005985324e-06,
      "logits/chosen": 6.069305896759033,
      "logits/rejected": 5.8950395584106445,
      "logps/chosen": -0.37205421924591064,
      "logps/rejected": -0.6190425753593445,
      "loss": 0.796,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5580812692642212,
      "rewards/margins": 0.3704826235771179,
      "rewards/rejected": -0.9285639524459839,
      "step": 220
    },
    {
      "epoch": 0.1856710393541877,
      "grad_norm": 0.4138280153274536,
      "learning_rate": 4.715508948078037e-06,
      "logits/chosen": 3.5271706581115723,
      "logits/rejected": 3.363534927368164,
      "logps/chosen": -0.41677650809288025,
      "logps/rejected": -0.7121980786323547,
      "loss": 0.7457,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6251648664474487,
      "rewards/margins": 0.4431324005126953,
      "rewards/rejected": -1.0682971477508545,
      "step": 230
    },
    {
      "epoch": 0.19374369323915236,
      "grad_norm": 1.4327284097671509,
      "learning_rate": 4.690766700109659e-06,
      "logits/chosen": 2.296924114227295,
      "logits/rejected": 1.6135867834091187,
      "logps/chosen": -0.506227970123291,
      "logps/rejected": -0.8080593943595886,
      "loss": 0.7453,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.7593418955802917,
      "rewards/margins": 0.4527471661567688,
      "rewards/rejected": -1.21208918094635,
      "step": 240
    },
    {
      "epoch": 0.20181634712411706,
      "grad_norm": 0.4413074553012848,
      "learning_rate": 4.665063509461098e-06,
      "logits/chosen": 1.4450469017028809,
      "logits/rejected": 0.40727120637893677,
      "logps/chosen": -0.4749869406223297,
      "logps/rejected": -0.9967275857925415,
      "loss": 0.704,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.712480366230011,
      "rewards/margins": 0.7826108932495117,
      "rewards/rejected": -1.4950913190841675,
      "step": 250
    },
    {
      "epoch": 0.20181634712411706,
      "eval_logits/chosen": 1.4614256620407104,
      "eval_logits/rejected": 0.6895493865013123,
      "eval_logps/chosen": -0.5038881301879883,
      "eval_logps/rejected": -0.973581850528717,
      "eval_loss": 0.6982013583183289,
      "eval_rewards/accuracies": 0.6138613820075989,
      "eval_rewards/chosen": -0.7558321952819824,
      "eval_rewards/margins": 0.7045406699180603,
      "eval_rewards/rejected": -1.460372805595398,
      "eval_runtime": 29.0993,
      "eval_samples_per_second": 27.526,
      "eval_steps_per_second": 3.471,
      "step": 250
    },
    {
      "epoch": 0.20988900100908173,
      "grad_norm": 0.734251856803894,
      "learning_rate": 4.638410650401267e-06,
      "logits/chosen": 2.4274039268493652,
      "logits/rejected": 1.498230218887329,
      "logps/chosen": -0.5549699068069458,
      "logps/rejected": -0.9348627328872681,
      "loss": 0.7149,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.8324548602104187,
      "rewards/margins": 0.5698392987251282,
      "rewards/rejected": -1.4022941589355469,
      "step": 260
    },
    {
      "epoch": 0.21796165489404642,
      "grad_norm": 0.4445085823535919,
      "learning_rate": 4.610819813755038e-06,
      "logits/chosen": 2.042858600616455,
      "logits/rejected": 1.1695036888122559,
      "logps/chosen": -0.5573975443840027,
      "logps/rejected": -1.15065598487854,
      "loss": 0.6811,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8360962867736816,
      "rewards/margins": 0.889887809753418,
      "rewards/rejected": -1.72598397731781,
      "step": 270
    },
    {
      "epoch": 0.2260343087790111,
      "grad_norm": 0.5875476598739624,
      "learning_rate": 4.582303101775249e-06,
      "logits/chosen": 1.3029518127441406,
      "logits/rejected": 0.3808741271495819,
      "logps/chosen": -0.6656169891357422,
      "logps/rejected": -1.5305824279785156,
      "loss": 0.637,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9984253644943237,
      "rewards/margins": 1.2974482774734497,
      "rewards/rejected": -2.2958736419677734,
      "step": 280
    },
    {
      "epoch": 0.2341069626639758,
      "grad_norm": 0.3814420998096466,
      "learning_rate": 4.55287302283426e-06,
      "logits/chosen": 2.0200212001800537,
      "logits/rejected": 1.2681838274002075,
      "logps/chosen": -0.6776013970375061,
      "logps/rejected": -1.3369777202606201,
      "loss": 0.63,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.016402006149292,
      "rewards/margins": 0.9890643358230591,
      "rewards/rejected": -2.0054664611816406,
      "step": 290
    },
    {
      "epoch": 0.24217961654894046,
      "grad_norm": 0.6442322731018066,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": 1.6268196105957031,
      "logits/rejected": 0.4954712390899658,
      "logps/chosen": -0.8117648363113403,
      "logps/rejected": -1.7705228328704834,
      "loss": 0.6104,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2176473140716553,
      "rewards/margins": 1.4381370544433594,
      "rewards/rejected": -2.6557843685150146,
      "step": 300
    },
    {
      "epoch": 0.24217961654894046,
      "eval_logits/chosen": 1.4003607034683228,
      "eval_logits/rejected": 0.46628010272979736,
      "eval_logps/chosen": -0.7612115740776062,
      "eval_logps/rejected": -1.6895866394042969,
      "eval_loss": 0.5781419277191162,
      "eval_rewards/accuracies": 0.6534653306007385,
      "eval_rewards/chosen": -1.141817331314087,
      "eval_rewards/margins": 1.392562747001648,
      "eval_rewards/rejected": -2.5343799591064453,
      "eval_runtime": 29.102,
      "eval_samples_per_second": 27.524,
      "eval_steps_per_second": 3.471,
      "step": 300
    },
    {
      "epoch": 0.25025227043390513,
      "grad_norm": 0.7889758944511414,
      "learning_rate": 4.491324795060491e-06,
      "logits/chosen": 1.1702089309692383,
      "logits/rejected": 0.354276180267334,
      "logps/chosen": -0.8091352581977844,
      "logps/rejected": -1.944819450378418,
      "loss": 0.5807,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2137027978897095,
      "rewards/margins": 1.7035261392593384,
      "rewards/rejected": -2.9172286987304688,
      "step": 310
    },
    {
      "epoch": 0.25832492431886983,
      "grad_norm": 1.410145878791809,
      "learning_rate": 4.4592336433146e-06,
      "logits/chosen": 2.0981459617614746,
      "logits/rejected": 1.2289329767227173,
      "logps/chosen": -0.8151634931564331,
      "logps/rejected": -1.977259874343872,
      "loss": 0.5007,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.222745418548584,
      "rewards/margins": 1.7431443929672241,
      "rewards/rejected": -2.9658896923065186,
      "step": 320
    },
    {
      "epoch": 0.26639757820383453,
      "grad_norm": 0.5564689040184021,
      "learning_rate": 4.426283106939474e-06,
      "logits/chosen": 1.7921768426895142,
      "logits/rejected": 0.7705962061882019,
      "logps/chosen": -0.9244564771652222,
      "logps/rejected": -2.5274672508239746,
      "loss": 0.4757,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3866846561431885,
      "rewards/margins": 2.4045166969299316,
      "rewards/rejected": -3.791201114654541,
      "step": 330
    },
    {
      "epoch": 0.27447023208879917,
      "grad_norm": 0.7554243803024292,
      "learning_rate": 4.3924876391293915e-06,
      "logits/chosen": 1.947997808456421,
      "logits/rejected": 0.9661592245101929,
      "logps/chosen": -0.9154227375984192,
      "logps/rejected": -3.0766491889953613,
      "loss": 0.4581,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3731342554092407,
      "rewards/margins": 3.24183988571167,
      "rewards/rejected": -4.614974021911621,
      "step": 340
    },
    {
      "epoch": 0.28254288597376387,
      "grad_norm": 2.936426877975464,
      "learning_rate": 4.357862063693486e-06,
      "logits/chosen": 1.2846364974975586,
      "logits/rejected": 0.9968118667602539,
      "logps/chosen": -1.0272481441497803,
      "logps/rejected": -2.6980624198913574,
      "loss": 0.4989,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.54087233543396,
      "rewards/margins": 2.506221294403076,
      "rewards/rejected": -4.047093868255615,
      "step": 350
    },
    {
      "epoch": 0.28254288597376387,
      "eval_logits/chosen": 1.841333031654358,
      "eval_logits/rejected": 1.1651691198349,
      "eval_logps/chosen": -0.9644113183021545,
      "eval_logps/rejected": -2.631535053253174,
      "eval_loss": 0.44154587388038635,
      "eval_rewards/accuracies": 0.6732673048973083,
      "eval_rewards/chosen": -1.4466170072555542,
      "eval_rewards/margins": 2.500684976577759,
      "eval_rewards/rejected": -3.9473025798797607,
      "eval_runtime": 29.1062,
      "eval_samples_per_second": 27.52,
      "eval_steps_per_second": 3.47,
      "step": 350
    },
    {
      "epoch": 0.29061553985872857,
      "grad_norm": 0.49919044971466064,
      "learning_rate": 4.322421568553529e-06,
      "logits/chosen": 1.9874728918075562,
      "logits/rejected": 1.464611291885376,
      "logps/chosen": -0.9963932037353516,
      "logps/rejected": -2.8804218769073486,
      "loss": 0.4373,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4945898056030273,
      "rewards/margins": 2.8260433673858643,
      "rewards/rejected": -4.320633411407471,
      "step": 360
    },
    {
      "epoch": 0.29868819374369326,
      "grad_norm": 1.4991668462753296,
      "learning_rate": 4.286181699082008e-06,
      "logits/chosen": 2.5421957969665527,
      "logits/rejected": 1.941922903060913,
      "logps/chosen": -1.0214306116104126,
      "logps/rejected": -3.2893283367156982,
      "loss": 0.438,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5321458578109741,
      "rewards/margins": 3.4018466472625732,
      "rewards/rejected": -4.933992862701416,
      "step": 370
    },
    {
      "epoch": 0.3067608476286579,
      "grad_norm": 1.933100938796997,
      "learning_rate": 4.249158351283414e-06,
      "logits/chosen": 1.80439031124115,
      "logits/rejected": 1.4421275854110718,
      "logps/chosen": -1.1270357370376587,
      "logps/rejected": -3.17500901222229,
      "loss": 0.4542,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.6905533075332642,
      "rewards/margins": 3.071959972381592,
      "rewards/rejected": -4.762513160705566,
      "step": 380
    },
    {
      "epoch": 0.3148335015136226,
      "grad_norm": 3.017254590988159,
      "learning_rate": 4.211367764821722e-06,
      "logits/chosen": 3.4090209007263184,
      "logits/rejected": 2.784639835357666,
      "logps/chosen": -1.1713608503341675,
      "logps/rejected": -3.326244831085205,
      "loss": 0.4816,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.757041335105896,
      "rewards/margins": 3.232325792312622,
      "rewards/rejected": -4.9893670082092285,
      "step": 390
    },
    {
      "epoch": 0.3229061553985873,
      "grad_norm": 0.5897337198257446,
      "learning_rate": 4.172826515897146e-06,
      "logits/chosen": 3.6071503162384033,
      "logits/rejected": 2.738395929336548,
      "logps/chosen": -1.133429765701294,
      "logps/rejected": -3.509474277496338,
      "loss": 0.4431,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.7001447677612305,
      "rewards/margins": 3.5640671253204346,
      "rewards/rejected": -5.264212131500244,
      "step": 400
    },
    {
      "epoch": 0.3229061553985873,
      "eval_logits/chosen": 2.893691062927246,
      "eval_logits/rejected": 2.259718656539917,
      "eval_logps/chosen": -1.0705492496490479,
      "eval_logps/rejected": -2.935904026031494,
      "eval_loss": 0.4156961143016815,
      "eval_rewards/accuracies": 0.6732673048973083,
      "eval_rewards/chosen": -1.6058237552642822,
      "eval_rewards/margins": 2.798032283782959,
      "eval_rewards/rejected": -4.403855800628662,
      "eval_runtime": 29.186,
      "eval_samples_per_second": 27.445,
      "eval_steps_per_second": 3.461,
      "step": 400
    },
    {
      "epoch": 0.33097880928355194,
      "grad_norm": 2.219709634780884,
      "learning_rate": 4.133551509975264e-06,
      "logits/chosen": 2.8940415382385254,
      "logits/rejected": 2.545257091522217,
      "logps/chosen": -1.1774877309799194,
      "logps/rejected": -3.4022269248962402,
      "loss": 0.427,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7662317752838135,
      "rewards/margins": 3.3371081352233887,
      "rewards/rejected": -5.1033406257629395,
      "step": 410
    },
    {
      "epoch": 0.33905146316851664,
      "grad_norm": 0.9891649484634399,
      "learning_rate": 4.093559974371725e-06,
      "logits/chosen": 3.180427074432373,
      "logits/rejected": 2.8354597091674805,
      "logps/chosen": -1.2984545230865479,
      "logps/rejected": -3.4927687644958496,
      "loss": 0.4384,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9476817846298218,
      "rewards/margins": 3.291471481323242,
      "rewards/rejected": -5.2391533851623535,
      "step": 420
    },
    {
      "epoch": 0.34712411705348134,
      "grad_norm": 0.4516288936138153,
      "learning_rate": 4.052869450695776e-06,
      "logits/chosen": 4.026360511779785,
      "logits/rejected": 3.3932456970214844,
      "logps/chosen": -1.1195746660232544,
      "logps/rejected": -3.4704582691192627,
      "loss": 0.3934,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.6793619394302368,
      "rewards/margins": 3.5263259410858154,
      "rewards/rejected": -5.205687522888184,
      "step": 430
    },
    {
      "epoch": 0.35519677093844604,
      "grad_norm": 0.5276560187339783,
      "learning_rate": 4.011497787155938e-06,
      "logits/chosen": 3.410902500152588,
      "logits/rejected": 2.906996965408325,
      "logps/chosen": -1.205904245376587,
      "logps/rejected": -3.7434115409851074,
      "loss": 0.4147,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.8088566064834595,
      "rewards/margins": 3.806260585784912,
      "rewards/rejected": -5.615117073059082,
      "step": 440
    },
    {
      "epoch": 0.3632694248234107,
      "grad_norm": 0.4435446560382843,
      "learning_rate": 3.969463130731183e-06,
      "logits/chosen": 3.7616615295410156,
      "logits/rejected": 3.4183363914489746,
      "logps/chosen": -1.2180852890014648,
      "logps/rejected": -3.865541934967041,
      "loss": 0.4354,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8271278142929077,
      "rewards/margins": 3.9711856842041016,
      "rewards/rejected": -5.798312664031982,
      "step": 450
    },
    {
      "epoch": 0.3632694248234107,
      "eval_logits/chosen": 3.55381178855896,
      "eval_logits/rejected": 3.01943039894104,
      "eval_logps/chosen": -1.3239827156066895,
      "eval_logps/rejected": -3.6477084159851074,
      "eval_loss": 0.3905698359012604,
      "eval_rewards/accuracies": 0.7029703259468079,
      "eval_rewards/chosen": -1.9859741926193237,
      "eval_rewards/margins": 3.485589027404785,
      "eval_rewards/rejected": -5.471563816070557,
      "eval_runtime": 29.0958,
      "eval_samples_per_second": 27.53,
      "eval_steps_per_second": 3.471,
      "step": 450
    },
    {
      "epoch": 0.3713420787083754,
      "grad_norm": 0.8419796824455261,
      "learning_rate": 3.92678391921108e-06,
      "logits/chosen": 3.3496861457824707,
      "logits/rejected": 3.235544204711914,
      "logps/chosen": -1.4899990558624268,
      "logps/rejected": -4.507351398468018,
      "loss": 0.4023,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.2349984645843506,
      "rewards/margins": 4.526029109954834,
      "rewards/rejected": -6.7610273361206055,
      "step": 460
    },
    {
      "epoch": 0.3794147325933401,
      "grad_norm": 1.2379052639007568,
      "learning_rate": 3.88347887310836e-06,
      "logits/chosen": 3.5375685691833496,
      "logits/rejected": 3.34586763381958,
      "logps/chosen": -1.5865886211395264,
      "logps/rejected": -4.887473106384277,
      "loss": 0.3979,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.379883289337158,
      "rewards/margins": 4.9513258934021,
      "rewards/rejected": -7.331210136413574,
      "step": 470
    },
    {
      "epoch": 0.3874873864783047,
      "grad_norm": 1.9969083070755005,
      "learning_rate": 3.839566987447492e-06,
      "logits/chosen": 3.3116583824157715,
      "logits/rejected": 2.9652373790740967,
      "logps/chosen": -2.0900139808654785,
      "logps/rejected": -4.921124458312988,
      "loss": 0.3356,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.135021209716797,
      "rewards/margins": 4.246665000915527,
      "rewards/rejected": -7.381686210632324,
      "step": 480
    },
    {
      "epoch": 0.3955600403632694,
      "grad_norm": 2.3405466079711914,
      "learning_rate": 3.795067523432826e-06,
      "logits/chosen": 2.542064666748047,
      "logits/rejected": 2.503020763397217,
      "logps/chosen": -2.4674277305603027,
      "logps/rejected": -5.703408718109131,
      "loss": 0.3193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.701141834259033,
      "rewards/margins": 4.853971004486084,
      "rewards/rejected": -8.555112838745117,
      "step": 490
    },
    {
      "epoch": 0.4036326942482341,
      "grad_norm": 1.4798164367675781,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": 3.79949688911438,
      "logits/rejected": 3.7085328102111816,
      "logps/chosen": -2.6355605125427246,
      "logps/rejected": -5.005209922790527,
      "loss": 0.3103,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.953340530395508,
      "rewards/margins": 3.554474353790283,
      "rewards/rejected": -7.507814884185791,
      "step": 500
    },
    {
      "epoch": 0.4036326942482341,
      "eval_logits/chosen": 3.31948184967041,
      "eval_logits/rejected": 2.9104762077331543,
      "eval_logps/chosen": -2.606774091720581,
      "eval_logps/rejected": -5.2280120849609375,
      "eval_loss": 0.3228474259376526,
      "eval_rewards/accuracies": 0.9306930899620056,
      "eval_rewards/chosen": -3.9101614952087402,
      "eval_rewards/margins": 3.931856870651245,
      "eval_rewards/rejected": -7.8420186042785645,
      "eval_runtime": 29.0975,
      "eval_samples_per_second": 27.528,
      "eval_steps_per_second": 3.471,
      "step": 500
    },
    {
      "epoch": 0.4117053481331988,
      "grad_norm": 1.4098985195159912,
      "learning_rate": 3.7043841852542884e-06,
      "logits/chosen": 2.69830584526062,
      "logits/rejected": 2.7101688385009766,
      "logps/chosen": -2.7831828594207764,
      "logps/rejected": -5.963825225830078,
      "loss": 0.2884,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.174774169921875,
      "rewards/margins": 4.7709641456604,
      "rewards/rejected": -8.945737838745117,
      "step": 510
    },
    {
      "epoch": 0.41977800201816345,
      "grad_norm": 3.457231283187866,
      "learning_rate": 3.658240087799655e-06,
      "logits/chosen": 3.335204601287842,
      "logits/rejected": 3.1935055255889893,
      "logps/chosen": -2.66495943069458,
      "logps/rejected": -5.2761640548706055,
      "loss": 0.3093,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.997438907623291,
      "rewards/margins": 3.916806697845459,
      "rewards/rejected": -7.91424560546875,
      "step": 520
    },
    {
      "epoch": 0.42785065590312815,
      "grad_norm": 1.7744654417037964,
      "learning_rate": 3.611587947962319e-06,
      "logits/chosen": 2.725924015045166,
      "logits/rejected": 2.605053186416626,
      "logps/chosen": -2.5286645889282227,
      "logps/rejected": -5.774144172668457,
      "loss": 0.2972,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.792996883392334,
      "rewards/margins": 4.868220329284668,
      "rewards/rejected": -8.661216735839844,
      "step": 530
    },
    {
      "epoch": 0.43592330978809285,
      "grad_norm": 2.863692283630371,
      "learning_rate": 3.564448228912682e-06,
      "logits/chosen": 2.304443120956421,
      "logits/rejected": 2.4836020469665527,
      "logps/chosen": -2.699148654937744,
      "logps/rejected": -6.340291500091553,
      "loss": 0.3136,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.048723220825195,
      "rewards/margins": 5.461714267730713,
      "rewards/rejected": -9.510437965393066,
      "step": 540
    },
    {
      "epoch": 0.4439959636730575,
      "grad_norm": 4.6681623458862305,
      "learning_rate": 3.516841607689501e-06,
      "logits/chosen": 3.1207594871520996,
      "logits/rejected": 2.947653293609619,
      "logps/chosen": -3.0062317848205566,
      "logps/rejected": -6.26816463470459,
      "loss": 0.2822,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.509347438812256,
      "rewards/margins": 4.892899513244629,
      "rewards/rejected": -9.402246475219727,
      "step": 550
    },
    {
      "epoch": 0.4439959636730575,
      "eval_logits/chosen": 2.879721164703369,
      "eval_logits/rejected": 2.52551007270813,
      "eval_logps/chosen": -2.6468751430511475,
      "eval_logps/rejected": -5.636789321899414,
      "eval_loss": 0.2939932942390442,
      "eval_rewards/accuracies": 0.9306930899620056,
      "eval_rewards/chosen": -3.9703128337860107,
      "eval_rewards/margins": 4.4848713874816895,
      "eval_rewards/rejected": -8.455184936523438,
      "eval_runtime": 29.4025,
      "eval_samples_per_second": 27.243,
      "eval_steps_per_second": 3.435,
      "step": 550
    },
    {
      "epoch": 0.4520686175580222,
      "grad_norm": 1.9975569248199463,
      "learning_rate": 3.4687889661302577e-06,
      "logits/chosen": 2.5885415077209473,
      "logits/rejected": 2.5374302864074707,
      "logps/chosen": -2.886643171310425,
      "logps/rejected": -6.636415004730225,
      "loss": 0.2736,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.329964637756348,
      "rewards/margins": 5.624656677246094,
      "rewards/rejected": -9.954621315002441,
      "step": 560
    },
    {
      "epoch": 0.4601412714429869,
      "grad_norm": 3.3965606689453125,
      "learning_rate": 3.4203113817116955e-06,
      "logits/chosen": 2.0093374252319336,
      "logits/rejected": 2.1064200401306152,
      "logps/chosen": -3.251981735229492,
      "logps/rejected": -6.749331474304199,
      "loss": 0.2776,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.877972602844238,
      "rewards/margins": 5.246025085449219,
      "rewards/rejected": -10.12399673461914,
      "step": 570
    },
    {
      "epoch": 0.4682139253279516,
      "grad_norm": 2.7079927921295166,
      "learning_rate": 3.3714301183045382e-06,
      "logits/chosen": 3.916841506958008,
      "logits/rejected": 3.524440288543701,
      "logps/chosen": -3.304469347000122,
      "logps/rejected": -7.1993279457092285,
      "loss": 0.2547,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.956704139709473,
      "rewards/margins": 5.842287540435791,
      "rewards/rejected": -10.798992156982422,
      "step": 580
    },
    {
      "epoch": 0.47628657921291623,
      "grad_norm": 7.504203796386719,
      "learning_rate": 3.3221666168464584e-06,
      "logits/chosen": 2.7350666522979736,
      "logits/rejected": 2.2590534687042236,
      "logps/chosen": -3.0954322814941406,
      "logps/rejected": -6.782465934753418,
      "loss": 0.2839,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.643147945404053,
      "rewards/margins": 5.530551433563232,
      "rewards/rejected": -10.173700332641602,
      "step": 590
    },
    {
      "epoch": 0.4843592330978809,
      "grad_norm": 2.8209569454193115,
      "learning_rate": 3.272542485937369e-06,
      "logits/chosen": 3.7743606567382812,
      "logits/rejected": 3.2480530738830566,
      "logps/chosen": -3.196892261505127,
      "logps/rejected": -6.42517614364624,
      "loss": 0.3237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.7953386306762695,
      "rewards/margins": 4.8424248695373535,
      "rewards/rejected": -9.637763977050781,
      "step": 600
    },
    {
      "epoch": 0.4843592330978809,
      "eval_logits/chosen": 2.6344597339630127,
      "eval_logits/rejected": 2.46531081199646,
      "eval_logps/chosen": -2.9877941608428955,
      "eval_logps/rejected": -6.266376972198486,
      "eval_loss": 0.2827170193195343,
      "eval_rewards/accuracies": 0.9603960514068604,
      "eval_rewards/chosen": -4.481690406799316,
      "eval_rewards/margins": 4.917874336242676,
      "eval_rewards/rejected": -9.399564743041992,
      "eval_runtime": 29.0997,
      "eval_samples_per_second": 27.526,
      "eval_steps_per_second": 3.471,
      "step": 600
    },
    {
      "epoch": 0.4924318869828456,
      "grad_norm": 7.072656631469727,
      "learning_rate": 3.222579492361179e-06,
      "logits/chosen": 3.2776103019714355,
      "logits/rejected": 3.166206121444702,
      "logps/chosen": -3.193716049194336,
      "logps/rejected": -6.185642242431641,
      "loss": 0.2986,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.790574073791504,
      "rewards/margins": 4.487889289855957,
      "rewards/rejected": -9.278463363647461,
      "step": 610
    },
    {
      "epoch": 0.5005045408678103,
      "grad_norm": 2.7188704013824463,
      "learning_rate": 3.1722995515381644e-06,
      "logits/chosen": 3.189532518386841,
      "logits/rejected": 2.9885735511779785,
      "logps/chosen": -3.3461785316467285,
      "logps/rejected": -6.720322608947754,
      "loss": 0.2776,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.019268035888672,
      "rewards/margins": 5.061214447021484,
      "rewards/rejected": -10.080483436584473,
      "step": 620
    },
    {
      "epoch": 0.508577194752775,
      "grad_norm": 2.6990387439727783,
      "learning_rate": 3.121724717912138e-06,
      "logits/chosen": 3.7751071453094482,
      "logits/rejected": 3.406925916671753,
      "logps/chosen": -3.3688464164733887,
      "logps/rejected": -6.132623672485352,
      "loss": 0.2688,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -5.053269386291504,
      "rewards/margins": 4.145665168762207,
      "rewards/rejected": -9.198934555053711,
      "step": 630
    },
    {
      "epoch": 0.5166498486377397,
      "grad_norm": 1.964301347732544,
      "learning_rate": 3.0708771752766397e-06,
      "logits/chosen": 2.6602158546447754,
      "logits/rejected": 2.162348508834839,
      "logps/chosen": -2.884718894958496,
      "logps/rejected": -6.869751930236816,
      "loss": 0.2435,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.327078342437744,
      "rewards/margins": 5.9775495529174805,
      "rewards/rejected": -10.304628372192383,
      "step": 640
    },
    {
      "epoch": 0.5247225025227044,
      "grad_norm": 3.201704740524292,
      "learning_rate": 3.019779227044398e-06,
      "logits/chosen": 3.83849835395813,
      "logits/rejected": 3.5917961597442627,
      "logps/chosen": -3.2030792236328125,
      "logps/rejected": -6.711284637451172,
      "loss": 0.2865,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.804618835449219,
      "rewards/margins": 5.262308120727539,
      "rewards/rejected": -10.066926956176758,
      "step": 650
    },
    {
      "epoch": 0.5247225025227044,
      "eval_logits/chosen": 2.6539251804351807,
      "eval_logits/rejected": 2.3508856296539307,
      "eval_logps/chosen": -3.0629732608795166,
      "eval_logps/rejected": -6.429784297943115,
      "eval_loss": 0.25649499893188477,
      "eval_rewards/accuracies": 0.9702970385551453,
      "eval_rewards/chosen": -4.594459533691406,
      "eval_rewards/margins": 5.0502166748046875,
      "eval_rewards/rejected": -9.644676208496094,
      "eval_runtime": 29.1031,
      "eval_samples_per_second": 27.523,
      "eval_steps_per_second": 3.47,
      "step": 650
    },
    {
      "epoch": 0.5327951564076691,
      "grad_norm": 3.517667055130005,
      "learning_rate": 2.9684532864643123e-06,
      "logits/chosen": 2.804810047149658,
      "logits/rejected": 2.8151614665985107,
      "logps/chosen": -3.4792568683624268,
      "logps/rejected": -7.08618688583374,
      "loss": 0.243,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.2188849449157715,
      "rewards/margins": 5.41039514541626,
      "rewards/rejected": -10.629280090332031,
      "step": 660
    },
    {
      "epoch": 0.5408678102926338,
      "grad_norm": 8.366011619567871,
      "learning_rate": 2.9169218667902562e-06,
      "logits/chosen": 2.1079938411712646,
      "logits/rejected": 2.2689383029937744,
      "logps/chosen": -3.3151912689208984,
      "logps/rejected": -7.454249382019043,
      "loss": 0.266,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.972787380218506,
      "rewards/margins": 6.208587169647217,
      "rewards/rejected": -11.181374549865723,
      "step": 670
    },
    {
      "epoch": 0.5489404641775983,
      "grad_norm": 4.161479473114014,
      "learning_rate": 2.8652075714060296e-06,
      "logits/chosen": 2.1858832836151123,
      "logits/rejected": 2.1688101291656494,
      "logps/chosen": -3.3753318786621094,
      "logps/rejected": -6.9990692138671875,
      "loss": 0.2356,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.0629987716674805,
      "rewards/margins": 5.435605525970459,
      "rewards/rejected": -10.498603820800781,
      "step": 680
    },
    {
      "epoch": 0.557013118062563,
      "grad_norm": 2.156621217727661,
      "learning_rate": 2.813333083910761e-06,
      "logits/chosen": 2.5004096031188965,
      "logits/rejected": 2.289215087890625,
      "logps/chosen": -3.711970567703247,
      "logps/rejected": -7.685518741607666,
      "loss": 0.2286,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.56795597076416,
      "rewards/margins": 5.960321426391602,
      "rewards/rejected": -11.528276443481445,
      "step": 690
    },
    {
      "epoch": 0.5650857719475277,
      "grad_norm": 4.547532081604004,
      "learning_rate": 2.761321158169134e-06,
      "logits/chosen": 3.112332820892334,
      "logits/rejected": 2.685454845428467,
      "logps/chosen": -3.510646343231201,
      "logps/rejected": -7.8125505447387695,
      "loss": 0.2222,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.265969276428223,
      "rewards/margins": 6.452856540679932,
      "rewards/rejected": -11.718826293945312,
      "step": 700
    },
    {
      "epoch": 0.5650857719475277,
      "eval_logits/chosen": 2.7667236328125,
      "eval_logits/rejected": 2.4983408451080322,
      "eval_logps/chosen": -3.292154550552368,
      "eval_logps/rejected": -6.814427375793457,
      "eval_loss": 0.24892139434814453,
      "eval_rewards/accuracies": 0.9702970385551453,
      "eval_rewards/chosen": -4.938231468200684,
      "eval_rewards/margins": 5.2834086418151855,
      "eval_rewards/rejected": -10.221639633178711,
      "eval_runtime": 29.0862,
      "eval_samples_per_second": 27.539,
      "eval_steps_per_second": 3.472,
      "step": 700
    },
    {
      "epoch": 0.5731584258324924,
      "grad_norm": 3.3842532634735107,
      "learning_rate": 2.70919460833079e-06,
      "logits/chosen": 2.6863622665405273,
      "logits/rejected": 2.7381534576416016,
      "logps/chosen": -3.1926321983337402,
      "logps/rejected": -7.823538780212402,
      "loss": 0.2153,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.788949012756348,
      "rewards/margins": 6.946360111236572,
      "rewards/rejected": -11.735308647155762,
      "step": 710
    },
    {
      "epoch": 0.5812310797174571,
      "grad_norm": 2.0056378841400146,
      "learning_rate": 2.6569762988232838e-06,
      "logits/chosen": 2.938141345977783,
      "logits/rejected": 2.8052525520324707,
      "logps/chosen": -3.4339518547058105,
      "logps/rejected": -7.353487968444824,
      "loss": 0.2422,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.150927543640137,
      "rewards/margins": 5.879303932189941,
      "rewards/rejected": -11.030232429504395,
      "step": 720
    },
    {
      "epoch": 0.5893037336024218,
      "grad_norm": 2.9281482696533203,
      "learning_rate": 2.604689134322999e-06,
      "logits/chosen": 3.5540454387664795,
      "logits/rejected": 3.6734695434570312,
      "logps/chosen": -3.0852925777435303,
      "logps/rejected": -6.869669437408447,
      "loss": 0.2694,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.627939224243164,
      "rewards/margins": 5.676565647125244,
      "rewards/rejected": -10.304505348205566,
      "step": 730
    },
    {
      "epoch": 0.5973763874873865,
      "grad_norm": 3.3841099739074707,
      "learning_rate": 2.5523560497083927e-06,
      "logits/chosen": 2.7171874046325684,
      "logits/rejected": 2.4911468029022217,
      "logps/chosen": -3.4943466186523438,
      "logps/rejected": -7.486982822418213,
      "loss": 0.2184,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.241519927978516,
      "rewards/margins": 5.988953113555908,
      "rewards/rejected": -11.230472564697266,
      "step": 740
    },
    {
      "epoch": 0.6054490413723511,
      "grad_norm": 2.6915197372436523,
      "learning_rate": 2.5e-06,
      "logits/chosen": 2.866816759109497,
      "logits/rejected": 2.84656023979187,
      "logps/chosen": -3.155074119567871,
      "logps/rejected": -8.019740104675293,
      "loss": 0.2364,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.732610702514648,
      "rewards/margins": 7.297000885009766,
      "rewards/rejected": -12.029611587524414,
      "step": 750
    },
    {
      "epoch": 0.6054490413723511,
      "eval_logits/chosen": 2.7122883796691895,
      "eval_logits/rejected": 2.4663355350494385,
      "eval_logps/chosen": -3.362666606903076,
      "eval_logps/rejected": -7.107527732849121,
      "eval_loss": 0.2374112904071808,
      "eval_rewards/accuracies": 0.9801980257034302,
      "eval_rewards/chosen": -5.043999671936035,
      "eval_rewards/margins": 5.617292881011963,
      "eval_rewards/rejected": -10.661293983459473,
      "eval_runtime": 29.0963,
      "eval_samples_per_second": 27.529,
      "eval_steps_per_second": 3.471,
      "step": 750
    }
  ],
  "logging_steps": 10,
  "max_steps": 1500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.7450483691657626e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}