{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.9618082618862043,
"eval_steps": 200,
"global_step": 7600,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.03897116134060795,
"grad_norm": 0.21226342022418976,
"learning_rate": 0.00025,
"logits/chosen": -19.687898635864258,
"logits/rejected": -18.633106231689453,
"logps/chosen": -343.7872619628906,
"logps/rejected": -250.41775512695312,
"loss": 0.4838,
"rewards/accuracies": 0.793749988079071,
"rewards/chosen": 0.6970043182373047,
"rewards/margins": 0.9049937725067139,
"rewards/rejected": -0.2079893946647644,
"step": 100
},
{
"epoch": 0.0779423226812159,
"grad_norm": 1.92564857006073,
"learning_rate": 0.0005,
"logits/chosen": -19.59960174560547,
"logits/rejected": -18.517173767089844,
"logps/chosen": -349.5354309082031,
"logps/rejected": -277.8677062988281,
"loss": 0.2445,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 1.4490649700164795,
"rewards/margins": 3.1850626468658447,
"rewards/rejected": -1.7359976768493652,
"step": 200
},
{
"epoch": 0.0779423226812159,
"eval_logits/chosen": -16.551204681396484,
"eval_logits/rejected": -16.51001739501953,
"eval_logps/chosen": -340.37274169921875,
"eval_logps/rejected": -285.42059326171875,
"eval_loss": 0.19902318716049194,
"eval_rewards/accuracies": 0.9711538553237915,
"eval_rewards/chosen": 0.12859317660331726,
"eval_rewards/margins": 1.9796644449234009,
"eval_rewards/rejected": -1.8510712385177612,
"eval_runtime": 183.9267,
"eval_samples_per_second": 1.131,
"eval_steps_per_second": 0.565,
"step": 200
},
{
"epoch": 0.11691348402182385,
"grad_norm": 1.4603872299194336,
"learning_rate": 0.0004997805905390462,
"logits/chosen": -19.685632705688477,
"logits/rejected": -18.630176544189453,
"logps/chosen": -347.72479248046875,
"logps/rejected": -297.4257507324219,
"loss": 0.195,
"rewards/accuracies": 0.8799999952316284,
"rewards/chosen": 0.857117235660553,
"rewards/margins": 4.945696830749512,
"rewards/rejected": -4.088580131530762,
"step": 300
},
{
"epoch": 0.1558846453624318,
"grad_norm": 2.9201486110687256,
"learning_rate": 0.0004991227472802768,
"logits/chosen": -19.715879440307617,
"logits/rejected": -18.654006958007812,
"logps/chosen": -347.0750427246094,
"logps/rejected": -297.565185546875,
"loss": 0.1965,
"rewards/accuracies": 0.8949999809265137,
"rewards/chosen": 0.27759745717048645,
"rewards/margins": 5.794647693634033,
"rewards/rejected": -5.517050266265869,
"step": 400
},
{
"epoch": 0.1558846453624318,
"eval_logits/chosen": -16.683916091918945,
"eval_logits/rejected": -16.602561950683594,
"eval_logps/chosen": -343.2335510253906,
"eval_logps/rejected": -309.0007629394531,
"eval_loss": 0.04625505581498146,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.15748834609985352,
"eval_rewards/margins": 4.051597595214844,
"eval_rewards/rejected": -4.2090864181518555,
"eval_runtime": 183.7705,
"eval_samples_per_second": 1.132,
"eval_steps_per_second": 0.566,
"step": 400
},
{
"epoch": 0.19485580670303976,
"grad_norm": 0.7862759232521057,
"learning_rate": 0.0004980276249199705,
"logits/chosen": -19.461082458496094,
"logits/rejected": -18.434755325317383,
"logps/chosen": -370.48394775390625,
"logps/rejected": -314.01043701171875,
"loss": 0.1873,
"rewards/accuracies": 0.8974999785423279,
"rewards/chosen": 0.1783198118209839,
"rewards/margins": 5.953300952911377,
"rewards/rejected": -5.774980545043945,
"step": 500
},
{
"epoch": 0.2338269680436477,
"grad_norm": 0.5820030570030212,
"learning_rate": 0.0004964971456997812,
"logits/chosen": -19.28722381591797,
"logits/rejected": -18.402751922607422,
"logps/chosen": -369.10980224609375,
"logps/rejected": -328.44403076171875,
"loss": 0.19,
"rewards/accuracies": 0.9024999737739563,
"rewards/chosen": -0.12349352240562439,
"rewards/margins": 6.251636505126953,
"rewards/rejected": -6.375130653381348,
"step": 600
},
{
"epoch": 0.2338269680436477,
"eval_logits/chosen": -16.357208251953125,
"eval_logits/rejected": -16.30510139465332,
"eval_logps/chosen": -336.88409423828125,
"eval_logps/rejected": -304.0970764160156,
"eval_loss": 0.036708563566207886,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.47745734453201294,
"eval_rewards/margins": 4.196179389953613,
"eval_rewards/rejected": -3.718721866607666,
"eval_runtime": 183.7701,
"eval_samples_per_second": 1.132,
"eval_steps_per_second": 0.566,
"step": 600
},
{
"epoch": 0.2727981293842556,
"grad_norm": 0.3981655538082123,
"learning_rate": 0.0004945339960326746,
"logits/chosen": -19.472816467285156,
"logits/rejected": -18.500709533691406,
"logps/chosen": -368.1885681152344,
"logps/rejected": -322.74298095703125,
"loss": 0.171,
"rewards/accuracies": 0.9012500047683716,
"rewards/chosen": -0.00018103599722962826,
"rewards/margins": 6.297857761383057,
"rewards/rejected": -6.298038959503174,
"step": 700
},
{
"epoch": 0.3117692907248636,
"grad_norm": 0.429470956325531,
"learning_rate": 0.0004921416217875326,
"logits/chosen": -19.675304412841797,
"logits/rejected": -18.701597213745117,
"logps/chosen": -354.1678771972656,
"logps/rejected": -312.2345886230469,
"loss": 0.1742,
"rewards/accuracies": 0.8912500143051147,
"rewards/chosen": 0.06210971996188164,
"rewards/margins": 6.645627498626709,
"rewards/rejected": -6.583518028259277,
"step": 800
},
{
"epoch": 0.3117692907248636,
"eval_logits/chosen": -16.51410484313965,
"eval_logits/rejected": -16.480632781982422,
"eval_logps/chosen": -338.15936279296875,
"eval_logps/rejected": -315.7767333984375,
"eval_loss": 0.023502787575125694,
"eval_rewards/accuracies": 0.995192289352417,
"eval_rewards/chosen": 0.34993284940719604,
"eval_rewards/margins": 5.236622333526611,
"eval_rewards/rejected": -4.886689186096191,
"eval_runtime": 183.2137,
"eval_samples_per_second": 1.135,
"eval_steps_per_second": 0.568,
"step": 800
},
{
"epoch": 0.35074045206547155,
"grad_norm": 0.3401203453540802,
"learning_rate": 0.0004893242222407031,
"logits/chosen": -19.789016723632812,
"logits/rejected": -18.94336700439453,
"logps/chosen": -343.6241760253906,
"logps/rejected": -323.7110595703125,
"loss": 0.1681,
"rewards/accuracies": 0.9087499976158142,
"rewards/chosen": 0.2733341455459595,
"rewards/margins": 6.843422889709473,
"rewards/rejected": -6.5700883865356445,
"step": 900
},
{
"epoch": 0.3897116134060795,
"grad_norm": 1.2149152755737305,
"learning_rate": 0.00048608674270511344,
"logits/chosen": -19.847705841064453,
"logits/rejected": -18.942218780517578,
"logps/chosen": -359.46551513671875,
"logps/rejected": -334.84136962890625,
"loss": 0.1605,
"rewards/accuracies": 0.90625,
"rewards/chosen": 0.1338280439376831,
"rewards/margins": 7.973731517791748,
"rewards/rejected": -7.839903354644775,
"step": 1000
},
{
"epoch": 0.3897116134060795,
"eval_logits/chosen": -16.609466552734375,
"eval_logits/rejected": -16.567523956298828,
"eval_logps/chosen": -334.4723205566406,
"eval_logps/rejected": -314.89141845703125,
"eval_loss": 0.010429470799863338,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.7186338901519775,
"eval_rewards/margins": 5.516786575317383,
"eval_rewards/rejected": -4.798152446746826,
"eval_runtime": 183.9441,
"eval_samples_per_second": 1.131,
"eval_steps_per_second": 0.565,
"step": 1000
},
{
"epoch": 0.4286827747466875,
"grad_norm": 15.479838371276855,
"learning_rate": 0.00048243486584988133,
"logits/chosen": -19.61993408203125,
"logits/rejected": -18.709421157836914,
"logps/chosen": -361.1015625,
"logps/rejected": -337.32342529296875,
"loss": 0.173,
"rewards/accuracies": 0.9037500023841858,
"rewards/chosen": -0.6482878923416138,
"rewards/margins": 7.439374923706055,
"rewards/rejected": -8.087662696838379,
"step": 1100
},
{
"epoch": 0.4676539360872954,
"grad_norm": 2.251624822616577,
"learning_rate": 0.00047837500172566503,
"logits/chosen": -19.768835067749023,
"logits/rejected": -18.762409210205078,
"logps/chosen": -365.1903076171875,
"logps/rejected": -328.1022644042969,
"loss": 0.19,
"rewards/accuracies": 0.9200000166893005,
"rewards/chosen": -0.6630336046218872,
"rewards/margins": 6.923609733581543,
"rewards/rejected": -7.586642265319824,
"step": 1200
},
{
"epoch": 0.4676539360872954,
"eval_logits/chosen": -16.668439865112305,
"eval_logits/rejected": -16.63960075378418,
"eval_logps/chosen": -348.08587646484375,
"eval_logps/rejected": -331.30828857421875,
"eval_loss": 0.013935078866779804,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.6427211761474609,
"eval_rewards/margins": 5.797122478485107,
"eval_rewards/rejected": -6.43984317779541,
"eval_runtime": 183.93,
"eval_samples_per_second": 1.131,
"eval_steps_per_second": 0.565,
"step": 1200
},
{
"epoch": 0.5066250974279034,
"grad_norm": 2.569606304168701,
"learning_rate": 0.00047391427651325704,
"logits/chosen": -20.112619400024414,
"logits/rejected": -19.097084045410156,
"logps/chosen": -361.98895263671875,
"logps/rejected": -335.0581970214844,
"loss": 0.1348,
"rewards/accuracies": 0.9312499761581421,
"rewards/chosen": -0.5764763355255127,
"rewards/margins": 7.914137363433838,
"rewards/rejected": -8.49061393737793,
"step": 1300
},
{
"epoch": 0.5455962587685113,
"grad_norm": 1.0773099660873413,
"learning_rate": 0.00046906052001517164,
"logits/chosen": -20.002775192260742,
"logits/rejected": -19.024526596069336,
"logps/chosen": -369.9784851074219,
"logps/rejected": -346.8582763671875,
"loss": 0.1693,
"rewards/accuracies": 0.9237499833106995,
"rewards/chosen": -1.2993569374084473,
"rewards/margins": 8.027504920959473,
"rewards/rejected": -9.326862335205078,
"step": 1400
},
{
"epoch": 0.5455962587685113,
"eval_logits/chosen": -16.792936325073242,
"eval_logits/rejected": -16.73811912536621,
"eval_logps/chosen": -335.2118835449219,
"eval_logps/rejected": -317.7545471191406,
"eval_loss": 0.019140072166919708,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.6446801424026489,
"eval_rewards/margins": 5.729149341583252,
"eval_rewards/rejected": -5.084469318389893,
"eval_runtime": 183.9981,
"eval_samples_per_second": 1.13,
"eval_steps_per_second": 0.565,
"step": 1400
},
{
"epoch": 0.5845674201091192,
"grad_norm": 3.4432373046875,
"learning_rate": 0.00046382225191218373,
"logits/chosen": -20.071659088134766,
"logits/rejected": -19.052783966064453,
"logps/chosen": -363.5613708496094,
"logps/rejected": -345.07879638671875,
"loss": 0.1425,
"rewards/accuracies": 0.9312499761581421,
"rewards/chosen": -0.769163966178894,
"rewards/margins": 8.191140174865723,
"rewards/rejected": -8.96030330657959,
"step": 1500
},
{
"epoch": 0.6235385814497272,
"grad_norm": 1.9013152122497559,
"learning_rate": 0.0004582086668089399,
"logits/chosen": -19.665212631225586,
"logits/rejected": -18.75629234313965,
"logps/chosen": -367.5525817871094,
"logps/rejected": -342.85516357421875,
"loss": 0.1263,
"rewards/accuracies": 0.9325000047683716,
"rewards/chosen": -0.8581268191337585,
"rewards/margins": 8.124137878417969,
"rewards/rejected": -8.982264518737793,
"step": 1600
},
{
"epoch": 0.6235385814497272,
"eval_logits/chosen": -16.445354461669922,
"eval_logits/rejected": -16.387128829956055,
"eval_logps/chosen": -327.33563232421875,
"eval_logps/rejected": -312.2492980957031,
"eval_loss": 0.0060209427028894424,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 1.4323018789291382,
"eval_rewards/margins": 5.966245174407959,
"eval_rewards/rejected": -4.5339436531066895,
"eval_runtime": 184.0733,
"eval_samples_per_second": 1.13,
"eval_steps_per_second": 0.565,
"step": 1600
},
{
"epoch": 0.6625097427903351,
"grad_norm": 0.8941299915313721,
"learning_rate": 0.0004522296180948922,
"logits/chosen": -19.424589157104492,
"logits/rejected": -18.495784759521484,
"logps/chosen": -374.550537109375,
"logps/rejected": -353.1033630371094,
"loss": 0.125,
"rewards/accuracies": 0.9387500286102295,
"rewards/chosen": -0.5644223690032959,
"rewards/margins": 8.560413360595703,
"rewards/rejected": -9.124835014343262,
"step": 1700
},
{
"epoch": 0.7014809041309431,
"grad_norm": 1.1238608360290527,
"learning_rate": 0.00044589560064888347,
"logits/chosen": -19.239290237426758,
"logits/rejected": -18.375873565673828,
"logps/chosen": -369.72027587890625,
"logps/rejected": -349.2558898925781,
"loss": 0.1232,
"rewards/accuracies": 0.9375,
"rewards/chosen": -0.9773778319358826,
"rewards/margins": 8.248481750488281,
"rewards/rejected": -9.225860595703125,
"step": 1800
},
{
"epoch": 0.7014809041309431,
"eval_logits/chosen": -16.073932647705078,
"eval_logits/rejected": -16.034406661987305,
"eval_logps/chosen": -343.9805908203125,
"eval_logps/rejected": -342.4121398925781,
"eval_loss": 0.005078889429569244,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.23218610882759094,
"eval_rewards/margins": 7.318041801452637,
"eval_rewards/rejected": -7.550227642059326,
"eval_runtime": 183.2955,
"eval_samples_per_second": 1.135,
"eval_steps_per_second": 0.567,
"step": 1800
},
{
"epoch": 0.7404520654715511,
"grad_norm": 3.126462697982788,
"learning_rate": 0.00043921773241774185,
"logits/chosen": -19.31768226623535,
"logits/rejected": -18.37215232849121,
"logps/chosen": -348.50390625,
"logps/rejected": -336.8278503417969,
"loss": 0.0923,
"rewards/accuracies": 0.9587500095367432,
"rewards/chosen": -0.21741238236427307,
"rewards/margins": 8.6088228225708,
"rewards/rejected": -8.826234817504883,
"step": 1900
},
{
"epoch": 0.779423226812159,
"grad_norm": 0.6362993121147156,
"learning_rate": 0.0004322077349012186,
"logits/chosen": -19.6059513092041,
"logits/rejected": -18.7484073638916,
"logps/chosen": -359.93206787109375,
"logps/rejected": -353.2133483886719,
"loss": 0.0983,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -1.2052465677261353,
"rewards/margins": 8.998382568359375,
"rewards/rejected": -10.203628540039062,
"step": 2000
},
{
"epoch": 0.779423226812159,
"eval_logits/chosen": -16.276912689208984,
"eval_logits/rejected": -16.23944854736328,
"eval_logps/chosen": -340.11883544921875,
"eval_logps/rejected": -336.9291687011719,
"eval_loss": 0.003970090765506029,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.15398484468460083,
"eval_rewards/margins": 7.155914783477783,
"eval_rewards/rejected": -7.001929759979248,
"eval_runtime": 184.065,
"eval_samples_per_second": 1.13,
"eval_steps_per_second": 0.565,
"step": 2000
},
{
"epoch": 0.818394388152767,
"grad_norm": 3.1369752883911133,
"learning_rate": 0.00042487791257752493,
"logits/chosen": -19.724271774291992,
"logits/rejected": -18.804405212402344,
"logps/chosen": -371.5325012207031,
"logps/rejected": -357.1531677246094,
"loss": 0.1049,
"rewards/accuracies": 0.9512500166893005,
"rewards/chosen": -1.3020814657211304,
"rewards/margins": 8.643760681152344,
"rewards/rejected": -9.945842742919922,
"step": 2100
},
{
"epoch": 0.857365549493375,
"grad_norm": 1.0814727544784546,
"learning_rate": 0.0004172411313055802,
"logits/chosen": -19.90325164794922,
"logits/rejected": -19.001785278320312,
"logps/chosen": -376.0804748535156,
"logps/rejected": -359.4775390625,
"loss": 0.0924,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -0.4394536316394806,
"rewards/margins": 8.659097671508789,
"rewards/rejected": -9.098551750183105,
"step": 2200
},
{
"epoch": 0.857365549493375,
"eval_logits/chosen": -16.71300506591797,
"eval_logits/rejected": -16.685504913330078,
"eval_logps/chosen": -334.0523986816406,
"eval_logps/rejected": -324.8212890625,
"eval_loss": 0.00510050356388092,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.760625958442688,
"eval_rewards/margins": 6.551769733428955,
"eval_rewards/rejected": -5.791143417358398,
"eval_runtime": 183.3621,
"eval_samples_per_second": 1.134,
"eval_steps_per_second": 0.567,
"step": 2200
},
{
"epoch": 0.8963367108339828,
"grad_norm": 2.7771525382995605,
"learning_rate": 0.0004093107957418828,
"logits/chosen": -19.735408782958984,
"logits/rejected": -18.8337459564209,
"logps/chosen": -367.6742858886719,
"logps/rejected": -356.5105895996094,
"loss": 0.0789,
"rewards/accuracies": 0.9612500071525574,
"rewards/chosen": -1.1817975044250488,
"rewards/margins": 9.191315650939941,
"rewards/rejected": -10.373113632202148,
"step": 2300
},
{
"epoch": 0.9353078721745908,
"grad_norm": 0.0016880702460184693,
"learning_rate": 0.0004011008258116425,
"logits/chosen": -19.32187271118164,
"logits/rejected": -18.563722610473633,
"logps/chosen": -373.0425109863281,
"logps/rejected": -362.72589111328125,
"loss": 0.0848,
"rewards/accuracies": 0.9649999737739563,
"rewards/chosen": -2.054009437561035,
"rewards/margins": 9.20674991607666,
"rewards/rejected": -11.260760307312012,
"step": 2400
},
{
"epoch": 0.9353078721745908,
"eval_logits/chosen": -16.159223556518555,
"eval_logits/rejected": -16.161327362060547,
"eval_logps/chosen": -337.83880615234375,
"eval_logps/rejected": -339.8623046875,
"eval_loss": 0.0020727256778627634,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.38198867440223694,
"eval_rewards/margins": 7.677234172821045,
"eval_rewards/rejected": -7.295245170593262,
"eval_runtime": 183.8954,
"eval_samples_per_second": 1.131,
"eval_steps_per_second": 0.566,
"step": 2400
},
{
"epoch": 0.9742790335151987,
"grad_norm": 1.488077998161316,
"learning_rate": 0.00039262563227547396,
"logits/chosen": -19.280136108398438,
"logits/rejected": -18.451894760131836,
"logps/chosen": -376.1581115722656,
"logps/rejected": -366.6534118652344,
"loss": 0.0662,
"rewards/accuracies": 0.9725000262260437,
"rewards/chosen": -1.176061987876892,
"rewards/margins": 8.959639549255371,
"rewards/rejected": -10.135702133178711,
"step": 2500
},
{
"epoch": 1.0132501948558068,
"grad_norm": 2.0352509021759033,
"learning_rate": 0.0003839000914345393,
"logits/chosen": -19.330242156982422,
"logits/rejected": -18.416221618652344,
"logps/chosen": -373.55523681640625,
"logps/rejected": -357.8737487792969,
"loss": 0.0373,
"rewards/accuracies": 0.9837499856948853,
"rewards/chosen": -0.7969868183135986,
"rewards/margins": 9.221331596374512,
"rewards/rejected": -10.018318176269531,
"step": 2600
},
{
"epoch": 1.0132501948558068,
"eval_logits/chosen": -15.664155006408691,
"eval_logits/rejected": -15.684382438659668,
"eval_logps/chosen": -348.93939208984375,
"eval_logps/rejected": -357.6224365234375,
"eval_loss": 0.001914023538120091,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.7280683517456055,
"eval_rewards/margins": 8.34318733215332,
"eval_rewards/rejected": -9.071255683898926,
"eval_runtime": 183.3393,
"eval_samples_per_second": 1.135,
"eval_steps_per_second": 0.567,
"step": 2600
},
{
"epoch": 1.0522213561964147,
"grad_norm": 1.1277427673339844,
"learning_rate": 0.00037493951901853797,
"logits/chosen": -19.345949172973633,
"logits/rejected": -18.49242401123047,
"logps/chosen": -365.28350830078125,
"logps/rejected": -361.580810546875,
"loss": 0.021,
"rewards/accuracies": 0.9937499761581421,
"rewards/chosen": -0.9318392872810364,
"rewards/margins": 9.834463119506836,
"rewards/rejected": -10.766302108764648,
"step": 2700
},
{
"epoch": 1.0911925175370225,
"grad_norm": 0.07178483158349991,
"learning_rate": 0.00036575964330237904,
"logits/chosen": -19.510316848754883,
"logits/rejected": -18.55198860168457,
"logps/chosen": -367.44598388671875,
"logps/rejected": -363.74078369140625,
"loss": 0.0151,
"rewards/accuracies": 0.9937499761581421,
"rewards/chosen": -1.267324447631836,
"rewards/margins": 10.227116584777832,
"rewards/rejected": -11.494441032409668,
"step": 2800
},
{
"epoch": 1.0911925175370225,
"eval_logits/chosen": -15.850824356079102,
"eval_logits/rejected": -15.864118576049805,
"eval_logps/chosen": -359.7252502441406,
"eval_logps/rejected": -381.7374572753906,
"eval_loss": 0.0014377026818692684,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -1.8066591024398804,
"eval_rewards/margins": 9.676095962524414,
"eval_rewards/rejected": -11.482755661010742,
"eval_runtime": 183.4769,
"eval_samples_per_second": 1.134,
"eval_steps_per_second": 0.567,
"step": 2800
},
{
"epoch": 1.1301636788776306,
"grad_norm": 1.4340250492095947,
"learning_rate": 0.00035637657749872255,
"logits/chosen": -19.317781448364258,
"logits/rejected": -18.438331604003906,
"logps/chosen": -375.9151306152344,
"logps/rejected": -366.2434387207031,
"loss": 0.0137,
"rewards/accuracies": 0.9962499737739563,
"rewards/chosen": -2.468919277191162,
"rewards/margins": 10.26855754852295,
"rewards/rejected": -12.737476348876953,
"step": 2900
},
{
"epoch": 1.1691348402182384,
"grad_norm": 0.2114144116640091,
"learning_rate": 0.00034680679147484916,
"logits/chosen": -19.166513442993164,
"logits/rejected": -18.25996208190918,
"logps/chosen": -371.7158508300781,
"logps/rejected": -375.24481201171875,
"loss": 0.0177,
"rewards/accuracies": 0.9925000071525574,
"rewards/chosen": -1.933343768119812,
"rewards/margins": 10.125692367553711,
"rewards/rejected": -12.059035301208496,
"step": 3000
},
{
"epoch": 1.1691348402182384,
"eval_logits/chosen": -16.009708404541016,
"eval_logits/rejected": -15.956798553466797,
"eval_logps/chosen": -332.91302490234375,
"eval_logps/rejected": -345.1927490234375,
"eval_loss": 0.0008918559760786593,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.874567449092865,
"eval_rewards/margins": 8.702858924865723,
"eval_rewards/rejected": -7.8282904624938965,
"eval_runtime": 184.0136,
"eval_samples_per_second": 1.13,
"eval_steps_per_second": 0.565,
"step": 3000
},
{
"epoch": 1.2081060015588465,
"grad_norm": 1.6861902475357056,
"learning_rate": 0.00033706708284350227,
"logits/chosen": -19.25012969970703,
"logits/rejected": -18.424623489379883,
"logps/chosen": -372.6192932128906,
"logps/rejected": -379.19549560546875,
"loss": 0.0209,
"rewards/accuracies": 0.9950000047683716,
"rewards/chosen": -1.3639909029006958,
"rewards/margins": 10.262293815612793,
"rewards/rejected": -11.626285552978516,
"step": 3100
},
{
"epoch": 1.2470771628994544,
"grad_norm": 0.16628094017505646,
"learning_rate": 0.00032717454747844735,
"logits/chosen": -19.219282150268555,
"logits/rejected": -18.238243103027344,
"logps/chosen": -365.1196594238281,
"logps/rejected": -364.3663330078125,
"loss": 0.0168,
"rewards/accuracies": 0.9912499785423279,
"rewards/chosen": -1.1836423873901367,
"rewards/margins": 10.353758811950684,
"rewards/rejected": -11.53740119934082,
"step": 3200
},
{
"epoch": 1.2470771628994544,
"eval_logits/chosen": -15.836195945739746,
"eval_logits/rejected": -15.786764144897461,
"eval_logps/chosen": -340.96075439453125,
"eval_logps/rejected": -357.6408996582031,
"eval_loss": 0.0008726614178158343,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.06979300826787949,
"eval_rewards/margins": 9.142892837524414,
"eval_rewards/rejected": -9.073100090026855,
"eval_runtime": 183.8457,
"eval_samples_per_second": 1.131,
"eval_steps_per_second": 0.566,
"step": 3200
},
{
"epoch": 1.2860483242400624,
"grad_norm": 0.0003638103371486068,
"learning_rate": 0.00031714654950649947,
"logits/chosen": -19.43454933166504,
"logits/rejected": -18.49201011657715,
"logps/chosen": -371.1318664550781,
"logps/rejected": -372.17938232421875,
"loss": 0.0163,
"rewards/accuracies": 0.9912499785423279,
"rewards/chosen": -1.4684104919433594,
"rewards/margins": 10.204544067382812,
"rewards/rejected": -11.672954559326172,
"step": 3300
},
{
"epoch": 1.3250194855806703,
"grad_norm": 0.0004742901655845344,
"learning_rate": 0.0003070006908286945,
"logits/chosen": -19.53050994873047,
"logits/rejected": -18.605304718017578,
"logps/chosen": -383.9638671875,
"logps/rejected": -382.642333984375,
"loss": 0.0159,
"rewards/accuracies": 0.9925000071525574,
"rewards/chosen": -1.5249592065811157,
"rewards/margins": 10.782021522521973,
"rewards/rejected": -12.306981086730957,
"step": 3400
},
{
"epoch": 1.3250194855806703,
"eval_logits/chosen": -15.98876953125,
"eval_logits/rejected": -15.942709922790527,
"eval_logps/chosen": -337.0641174316406,
"eval_logps/rejected": -357.69342041015625,
"eval_loss": 0.0007918892079032958,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.4594591557979584,
"eval_rewards/margins": 9.53781795501709,
"eval_rewards/rejected": -9.078359603881836,
"eval_runtime": 183.8285,
"eval_samples_per_second": 1.131,
"eval_steps_per_second": 0.566,
"step": 3400
},
{
"epoch": 1.3639906469212781,
"grad_norm": 3.363085985183716,
"learning_rate": 0.0002967547802240997,
"logits/chosen": -19.533775329589844,
"logits/rejected": -18.48381233215332,
"logps/chosen": -369.41827392578125,
"logps/rejected": -367.9228820800781,
"loss": 0.0152,
"rewards/accuracies": 0.9950000047683716,
"rewards/chosen": -0.9302732944488525,
"rewards/margins": 10.337385177612305,
"rewards/rejected": -11.267658233642578,
"step": 3500
},
{
"epoch": 1.4029618082618862,
"grad_norm": 0.014138066209852695,
"learning_rate": 0.00028642680209049715,
"logits/chosen": -19.569242477416992,
"logits/rejected": -18.72142791748047,
"logps/chosen": -355.16339111328125,
"logps/rejected": -369.4004211425781,
"loss": 0.0106,
"rewards/accuracies": 0.9987499713897705,
"rewards/chosen": -0.6573285460472107,
"rewards/margins": 10.500960350036621,
"rewards/rejected": -11.15829086303711,
"step": 3600
},
{
"epoch": 1.4029618082618862,
"eval_logits/chosen": -16.05227279663086,
"eval_logits/rejected": -16.000469207763672,
"eval_logps/chosen": -334.4986572265625,
"eval_logps/rejected": -352.8683166503906,
"eval_loss": 0.0009424146264791489,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.7160030603408813,
"eval_rewards/margins": 9.311846733093262,
"eval_rewards/rejected": -8.595843315124512,
"eval_runtime": 183.6848,
"eval_samples_per_second": 1.132,
"eval_steps_per_second": 0.566,
"step": 3600
},
{
"epoch": 1.4419329696024943,
"grad_norm": 0.009088713675737381,
"learning_rate": 0.00027603488487680684,
"logits/chosen": -19.91989517211914,
"logits/rejected": -18.872535705566406,
"logps/chosen": -372.661865234375,
"logps/rejected": -367.7705078125,
"loss": 0.0184,
"rewards/accuracies": 0.9937499761581421,
"rewards/chosen": -1.1601972579956055,
"rewards/margins": 10.311330795288086,
"rewards/rejected": -11.471526145935059,
"step": 3700
},
{
"epoch": 1.4809041309431021,
"grad_norm": 0.010365006513893604,
"learning_rate": 0.00026559726926266204,
"logits/chosen": -19.595075607299805,
"logits/rejected": -18.658893585205078,
"logps/chosen": -390.93206787109375,
"logps/rejected": -393.7774963378906,
"loss": 0.0095,
"rewards/accuracies": 0.9950000047683716,
"rewards/chosen": -2.7839019298553467,
"rewards/margins": 10.808964729309082,
"rewards/rejected": -13.592867851257324,
"step": 3800
},
{
"epoch": 1.4809041309431021,
"eval_logits/chosen": -15.648859977722168,
"eval_logits/rejected": -15.611715316772461,
"eval_logps/chosen": -346.69561767578125,
"eval_logps/rejected": -376.822265625,
"eval_loss": 0.0005593308596871793,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.5036950707435608,
"eval_rewards/margins": 10.487546920776367,
"eval_rewards/rejected": -10.991241455078125,
"eval_runtime": 184.1015,
"eval_samples_per_second": 1.13,
"eval_steps_per_second": 0.565,
"step": 3800
},
{
"epoch": 1.51987529228371,
"grad_norm": 0.12639912962913513,
"learning_rate": 0.00025513227614098707,
"logits/chosen": -19.499780654907227,
"logits/rejected": -18.518644332885742,
"logps/chosen": -366.645263671875,
"logps/rejected": -380.7431640625,
"loss": 0.0067,
"rewards/accuracies": 0.9975000023841858,
"rewards/chosen": -1.6605476140975952,
"rewards/margins": 10.881020545959473,
"rewards/rejected": -12.541565895080566,
"step": 3900
},
{
"epoch": 1.558846453624318,
"grad_norm": 0.32782620191574097,
"learning_rate": 0.00024465827445977964,
"logits/chosen": -19.350522994995117,
"logits/rejected": -18.31049346923828,
"logps/chosen": -380.0142822265625,
"logps/rejected": -377.0948791503906,
"loss": 0.0107,
"rewards/accuracies": 0.9962499737739563,
"rewards/chosen": -1.4461276531219482,
"rewards/margins": 10.850979804992676,
"rewards/rejected": -12.297107696533203,
"step": 4000
},
{
"epoch": 1.558846453624318,
"eval_logits/chosen": -15.838950157165527,
"eval_logits/rejected": -15.806854248046875,
"eval_logps/chosen": -336.4289245605469,
"eval_logps/rejected": -364.8307189941406,
"eval_loss": 0.00040981665370054543,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 0.5229762196540833,
"eval_rewards/margins": 10.315059661865234,
"eval_rewards/rejected": -9.792083740234375,
"eval_runtime": 184.0487,
"eval_samples_per_second": 1.13,
"eval_steps_per_second": 0.565,
"step": 4000
},
{
"epoch": 1.597817614964926,
"grad_norm": 1.2895886898040771,
"learning_rate": 0.00023419364897954285,
"logits/chosen": -19.494075775146484,
"logits/rejected": -18.560813903808594,
"logps/chosen": -361.8075256347656,
"logps/rejected": -380.0975646972656,
"loss": 0.0108,
"rewards/accuracies": 0.9950000047683716,
"rewards/chosen": -1.1338391304016113,
"rewards/margins": 11.00627326965332,
"rewards/rejected": -12.140111923217773,
"step": 4100
},
{
"epoch": 1.6367887763055338,
"grad_norm": 0.00033555322443135083,
"learning_rate": 0.00022375676800296247,
"logits/chosen": -19.587282180786133,
"logits/rejected": -18.464643478393555,
"logps/chosen": -379.6680603027344,
"logps/rejected": -372.1777648925781,
"loss": 0.0124,
"rewards/accuracies": 0.9937499761581421,
"rewards/chosen": -1.0544594526290894,
"rewards/margins": 11.037619590759277,
"rewards/rejected": -12.092079162597656,
"step": 4200
},
{
"epoch": 1.6367887763055338,
"eval_logits/chosen": -15.83126163482666,
"eval_logits/rejected": -15.794379234313965,
"eval_logps/chosen": -330.6876525878906,
"eval_logps/rejected": -356.1661682128906,
"eval_loss": 0.0004220473056193441,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 1.0971003770828247,
"eval_rewards/margins": 10.022727012634277,
"eval_rewards/rejected": -8.925626754760742,
"eval_runtime": 183.9983,
"eval_samples_per_second": 1.13,
"eval_steps_per_second": 0.565,
"step": 4200
},
{
"epoch": 1.6757599376461418,
"grad_norm": 1.2421733140945435,
"learning_rate": 0.00021336595113347144,
"logits/chosen": -19.402978897094727,
"logits/rejected": -18.351083755493164,
"logps/chosen": -370.1186828613281,
"logps/rejected": -374.5020446777344,
"loss": 0.009,
"rewards/accuracies": 0.9937499761581421,
"rewards/chosen": -1.1320860385894775,
"rewards/margins": 10.839487075805664,
"rewards/rejected": -11.971571922302246,
"step": 4300
},
{
"epoch": 1.71473109898675,
"grad_norm": 0.16441740095615387,
"learning_rate": 0.00020303943711929526,
"logits/chosen": -19.135257720947266,
"logits/rejected": -18.16560173034668,
"logps/chosen": -375.6964111328125,
"logps/rejected": -390.9460144042969,
"loss": 0.021,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.0195655822753906,
"rewards/margins": 11.327723503112793,
"rewards/rejected": -13.3472900390625,
"step": 4400
},
{
"epoch": 1.71473109898675,
"eval_logits/chosen": -15.488088607788086,
"eval_logits/rejected": -15.465119361877441,
"eval_logps/chosen": -344.2342529296875,
"eval_logps/rejected": -376.2817687988281,
"eval_loss": 0.0005891394102945924,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.25755709409713745,
"eval_rewards/margins": 10.679630279541016,
"eval_rewards/rejected": -10.937186241149902,
"eval_runtime": 184.1206,
"eval_samples_per_second": 1.13,
"eval_steps_per_second": 0.565,
"step": 4400
},
{
"epoch": 1.7537022603273578,
"grad_norm": 0.02925615757703781,
"learning_rate": 0.00019279535183942101,
"logits/chosen": -19.17304039001465,
"logits/rejected": -18.184049606323242,
"logps/chosen": -385.8622131347656,
"logps/rejected": -391.7120361328125,
"loss": 0.0083,
"rewards/accuracies": 0.9937499761581421,
"rewards/chosen": -1.9831608533859253,
"rewards/margins": 11.25387954711914,
"rewards/rejected": -13.237038612365723,
"step": 4500
},
{
"epoch": 1.7926734216679656,
"grad_norm": 0.05078015476465225,
"learning_rate": 0.00018265167648768259,
"logits/chosen": -19.064685821533203,
"logits/rejected": -18.237638473510742,
"logps/chosen": -376.9311218261719,
"logps/rejected": -387.96905517578125,
"loss": 0.0061,
"rewards/accuracies": 0.9950000047683716,
"rewards/chosen": -2.660050392150879,
"rewards/margins": 11.354269027709961,
"rewards/rejected": -14.014320373535156,
"step": 4600
},
{
"epoch": 1.7926734216679656,
"eval_logits/chosen": -15.26915454864502,
"eval_logits/rejected": -15.260515213012695,
"eval_logps/chosen": -355.8375549316406,
"eval_logps/rejected": -390.41119384765625,
"eval_loss": 0.0009181927889585495,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -1.417887568473816,
"eval_rewards/margins": 10.932249069213867,
"eval_rewards/rejected": -12.350136756896973,
"eval_runtime": 183.689,
"eval_samples_per_second": 1.132,
"eval_steps_per_second": 0.566,
"step": 4600
},
{
"epoch": 1.8316445830085737,
"grad_norm": 0.2935536205768585,
"learning_rate": 0.00017262621601080811,
"logits/chosen": -19.163440704345703,
"logits/rejected": -18.167402267456055,
"logps/chosen": -379.1352233886719,
"logps/rejected": -383.6081237792969,
"loss": 0.0126,
"rewards/accuracies": 0.9950000047683716,
"rewards/chosen": -2.3724796772003174,
"rewards/margins": 11.351487159729004,
"rewards/rejected": -13.723965644836426,
"step": 4700
},
{
"epoch": 1.8706157443491818,
"grad_norm": 0.02069302648305893,
"learning_rate": 0.00016273656785582984,
"logits/chosen": -18.975814819335938,
"logits/rejected": -18.142276763916016,
"logps/chosen": -383.4511413574219,
"logps/rejected": -393.5997009277344,
"loss": 0.0102,
"rewards/accuracies": 0.9950000047683716,
"rewards/chosen": -2.359323024749756,
"rewards/margins": 11.0841064453125,
"rewards/rejected": -13.443428993225098,
"step": 4800
},
{
"epoch": 1.8706157443491818,
"eval_logits/chosen": -15.47636604309082,
"eval_logits/rejected": -15.429464340209961,
"eval_logps/chosen": -341.96356201171875,
"eval_logps/rejected": -369.5177307128906,
"eval_loss": 0.0004053489537909627,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.030484315007925034,
"eval_rewards/margins": 10.230300903320312,
"eval_rewards/rejected": -10.260786056518555,
"eval_runtime": 183.2032,
"eval_samples_per_second": 1.135,
"eval_steps_per_second": 0.568,
"step": 4800
},
{
"epoch": 1.9095869056897894,
"grad_norm": 0.06283093243837357,
"learning_rate": 0.00015300009108171347,
"logits/chosen": -19.286584854125977,
"logits/rejected": -18.341442108154297,
"logps/chosen": -376.3005065917969,
"logps/rejected": -383.4756774902344,
"loss": 0.0044,
"rewards/accuracies": 0.9987499713897705,
"rewards/chosen": -1.3465908765792847,
"rewards/margins": 11.345978736877441,
"rewards/rejected": -12.692569732666016,
"step": 4900
},
{
"epoch": 1.9485580670303975,
"grad_norm": 0.04220004007220268,
"learning_rate": 0.00014343387588942392,
"logits/chosen": -19.211626052856445,
"logits/rejected": -18.220531463623047,
"logps/chosen": -389.9902648925781,
"logps/rejected": -397.5294494628906,
"loss": 0.0057,
"rewards/accuracies": 0.9962499737739563,
"rewards/chosen": -1.984826922416687,
"rewards/margins": 11.562482833862305,
"rewards/rejected": -13.547309875488281,
"step": 5000
},
{
"epoch": 1.9485580670303975,
"eval_logits/chosen": -15.311283111572266,
"eval_logits/rejected": -15.284743309020996,
"eval_logps/chosen": -350.1474304199219,
"eval_logps/rejected": -386.7135314941406,
"eval_loss": 0.00032912034657783806,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.8488726019859314,
"eval_rewards/margins": 11.131490707397461,
"eval_rewards/rejected": -11.980363845825195,
"eval_runtime": 183.6747,
"eval_samples_per_second": 1.132,
"eval_steps_per_second": 0.566,
"step": 5000
},
{
"epoch": 1.9875292283710055,
"grad_norm": 0.05133282020688057,
"learning_rate": 0.00013405471362391068,
"logits/chosen": -19.146968841552734,
"logits/rejected": -18.36100196838379,
"logps/chosen": -375.5859680175781,
"logps/rejected": -405.4987487792969,
"loss": 0.0055,
"rewards/accuracies": 0.9962499737739563,
"rewards/chosen": -2.0062613487243652,
"rewards/margins": 11.725434303283691,
"rewards/rejected": -13.731695175170898,
"step": 5100
},
{
"epoch": 2.0265003897116136,
"grad_norm": 0.003434022655710578,
"learning_rate": 0.00012487906730066888,
"logits/chosen": -19.30167579650879,
"logits/rejected": -18.223770141601562,
"logps/chosen": -376.9130859375,
"logps/rejected": -380.2550354003906,
"loss": 0.0025,
"rewards/accuracies": 0.9975000023841858,
"rewards/chosen": -1.750290870666504,
"rewards/margins": 11.900991439819336,
"rewards/rejected": -13.651283264160156,
"step": 5200
},
{
"epoch": 2.0265003897116136,
"eval_logits/chosen": -15.498884201049805,
"eval_logits/rejected": -15.481465339660645,
"eval_logps/chosen": -349.6870422363281,
"eval_logps/rejected": -385.3147888183594,
"eval_loss": 0.0003257237549405545,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.8028372526168823,
"eval_rewards/margins": 11.037657737731934,
"eval_rewards/rejected": -11.840494155883789,
"eval_runtime": 183.7823,
"eval_samples_per_second": 1.132,
"eval_steps_per_second": 0.566,
"step": 5200
},
{
"epoch": 2.0654715510522212,
"grad_norm": 0.026280131191015244,
"learning_rate": 0.00011592304270860795,
"logits/chosen": -19.42230796813965,
"logits/rejected": -18.394390106201172,
"logps/chosen": -400.98980712890625,
"logps/rejected": -409.95098876953125,
"loss": 0.005,
"rewards/accuracies": 0.9962499737739563,
"rewards/chosen": -2.090318202972412,
"rewards/margins": 11.665027618408203,
"rewards/rejected": -13.755346298217773,
"step": 5300
},
{
"epoch": 2.1044427123928293,
"grad_norm": 0.00010202277189819142,
"learning_rate": 0.00010720236013995221,
"logits/chosen": -19.477603912353516,
"logits/rejected": -18.485740661621094,
"logps/chosen": -365.8956298828125,
"logps/rejected": -397.5658264160156,
"loss": 0.0015,
"rewards/accuracies": 0.9987499713897705,
"rewards/chosen": -1.8521977663040161,
"rewards/margins": 12.37298583984375,
"rewards/rejected": -14.225183486938477,
"step": 5400
},
{
"epoch": 2.1044427123928293,
"eval_logits/chosen": -15.460063934326172,
"eval_logits/rejected": -15.440217018127441,
"eval_logps/chosen": -351.2416687011719,
"eval_logps/rejected": -389.2104797363281,
"eval_loss": 0.00034331483766436577,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.958295464515686,
"eval_rewards/margins": 11.27176570892334,
"eval_rewards/rejected": -12.230061531066895,
"eval_runtime": 183.6747,
"eval_samples_per_second": 1.132,
"eval_steps_per_second": 0.566,
"step": 5400
},
{
"epoch": 2.1434138737334374,
"grad_norm": 0.005236598663032055,
"learning_rate": 9.873232679679392e-05,
"logits/chosen": -19.37502670288086,
"logits/rejected": -18.3719482421875,
"logps/chosen": -381.2720947265625,
"logps/rejected": -403.01593017578125,
"loss": 0.0024,
"rewards/accuracies": 0.9975000023841858,
"rewards/chosen": -2.047292709350586,
"rewards/margins": 12.199724197387695,
"rewards/rejected": -14.247018814086914,
"step": 5500
},
{
"epoch": 2.182385035074045,
"grad_norm": 0.029128100723028183,
"learning_rate": 9.052780992273379e-05,
"logits/chosen": -19.223974227905273,
"logits/rejected": -18.315275192260742,
"logps/chosen": -378.5389404296875,
"logps/rejected": -399.8840637207031,
"loss": 0.0021,
"rewards/accuracies": 0.9975000023841858,
"rewards/chosen": -1.8436551094055176,
"rewards/margins": 12.235860824584961,
"rewards/rejected": -14.07951545715332,
"step": 5600
},
{
"epoch": 2.182385035074045,
"eval_logits/chosen": -15.46290111541748,
"eval_logits/rejected": -15.436098098754883,
"eval_logps/chosen": -349.15423583984375,
"eval_logps/rejected": -387.6983947753906,
"eval_loss": 0.00030186420190148056,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.7495561242103577,
"eval_rewards/margins": 11.329290390014648,
"eval_rewards/rejected": -12.078847885131836,
"eval_runtime": 183.2993,
"eval_samples_per_second": 1.135,
"eval_steps_per_second": 0.567,
"step": 5600
},
{
"epoch": 2.221356196414653,
"grad_norm": 0.011133521795272827,
"learning_rate": 8.26032107067696e-05,
"logits/chosen": -19.451305389404297,
"logits/rejected": -18.38578224182129,
"logps/chosen": -374.8850402832031,
"logps/rejected": -388.83575439453125,
"loss": 0.0014,
"rewards/accuracies": 0.9987499713897705,
"rewards/chosen": -1.683659315109253,
"rewards/margins": 12.163559913635254,
"rewards/rejected": -13.847216606140137,
"step": 5700
},
{
"epoch": 2.260327357755261,
"grad_norm": 7.387495134025812e-05,
"learning_rate": 7.497243900523937e-05,
"logits/chosen": -19.474388122558594,
"logits/rejected": -18.39251708984375,
"logps/chosen": -382.3981628417969,
"logps/rejected": -390.441162109375,
"loss": 0.0017,
"rewards/accuracies": 0.9987499713897705,
"rewards/chosen": -1.4145100116729736,
"rewards/margins": 11.843048095703125,
"rewards/rejected": -13.257555961608887,
"step": 5800
},
{
"epoch": 2.260327357755261,
"eval_logits/chosen": -15.475095748901367,
"eval_logits/rejected": -15.438820838928223,
"eval_logps/chosen": -347.4687805175781,
"eval_logps/rejected": -385.78497314453125,
"eval_loss": 0.0002713102730922401,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.5810099244117737,
"eval_rewards/margins": 11.306499481201172,
"eval_rewards/rejected": -11.887508392333984,
"eval_runtime": 183.4323,
"eval_samples_per_second": 1.134,
"eval_steps_per_second": 0.567,
"step": 5800
},
{
"epoch": 2.2992985190958692,
"grad_norm": 0.003995438106358051,
"learning_rate": 6.764888892618867e-05,
"logits/chosen": -19.462810516357422,
"logits/rejected": -18.417314529418945,
"logps/chosen": -375.923583984375,
"logps/rejected": -394.6671447753906,
"loss": 0.0051,
"rewards/accuracies": 0.9937499761581421,
"rewards/chosen": -1.9151861667633057,
"rewards/margins": 11.908782005310059,
"rewards/rejected": -13.823966026306152,
"step": 5900
},
{
"epoch": 2.338269680436477,
"grad_norm": 0.003454476362094283,
"learning_rate": 6.064541531901838e-05,
"logits/chosen": -19.41205406188965,
"logits/rejected": -18.35369873046875,
"logps/chosen": -374.3619384765625,
"logps/rejected": -392.4114990234375,
"loss": 0.004,
"rewards/accuracies": 0.9962499737739563,
"rewards/chosen": -2.0450844764709473,
"rewards/margins": 11.8558988571167,
"rewards/rejected": -13.900982856750488,
"step": 6000
},
{
"epoch": 2.338269680436477,
"eval_logits/chosen": -15.477252960205078,
"eval_logits/rejected": -15.445527076721191,
"eval_logps/chosen": -347.4806213378906,
"eval_logps/rejected": -386.5325927734375,
"eval_loss": 0.00025467213708907366,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.5821936130523682,
"eval_rewards/margins": 11.38007926940918,
"eval_rewards/rejected": -11.962273597717285,
"eval_runtime": 183.2231,
"eval_samples_per_second": 1.135,
"eval_steps_per_second": 0.568,
"step": 6000
},
{
"epoch": 2.377240841777085,
"grad_norm": 0.0025488168466836214,
"learning_rate": 5.397431121068011e-05,
"logits/chosen": -19.279691696166992,
"logits/rejected": -18.215682983398438,
"logps/chosen": -391.84844970703125,
"logps/rejected": -398.8818359375,
"loss": 0.003,
"rewards/accuracies": 0.9962499737739563,
"rewards/chosen": -2.172652006149292,
"rewards/margins": 12.266448020935059,
"rewards/rejected": -14.439099311828613,
"step": 6100
},
{
"epoch": 2.416212003117693,
"grad_norm": 0.0001306094927713275,
"learning_rate": 4.7647286228024944e-05,
"logits/chosen": -19.264507293701172,
"logits/rejected": -18.2991886138916,
"logps/chosen": -368.3695373535156,
"logps/rejected": -395.1746826171875,
"loss": 0.0021,
"rewards/accuracies": 0.9975000023841858,
"rewards/chosen": -2.2215933799743652,
"rewards/margins": 12.452881813049316,
"rewards/rejected": -14.674474716186523,
"step": 6200
},
{
"epoch": 2.416212003117693,
"eval_logits/chosen": -15.429259300231934,
"eval_logits/rejected": -15.400547981262207,
"eval_logps/chosen": -350.2945556640625,
"eval_logps/rejected": -390.7493896484375,
"eval_loss": 0.0002859699307009578,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.8635876178741455,
"eval_rewards/margins": 11.520365715026855,
"eval_rewards/rejected": -12.383952140808105,
"eval_runtime": 183.7971,
"eval_samples_per_second": 1.132,
"eval_steps_per_second": 0.566,
"step": 6200
},
{
"epoch": 2.455183164458301,
"grad_norm": 0.0013745080213993788,
"learning_rate": 4.167544604418003e-05,
"logits/chosen": -19.316665649414062,
"logits/rejected": -18.3441162109375,
"logps/chosen": -378.2589416503906,
"logps/rejected": -407.154052734375,
"loss": 0.0012,
"rewards/accuracies": 0.9987499713897705,
"rewards/chosen": -2.0685226917266846,
"rewards/margins": 12.507061958312988,
"rewards/rejected": -14.575584411621094,
"step": 6300
},
{
"epoch": 2.4941543257989087,
"grad_norm": 0.002951019676402211,
"learning_rate": 3.6069272885030256e-05,
"logits/chosen": -19.417644500732422,
"logits/rejected": -18.45380210876465,
"logps/chosen": -384.16253662109375,
"logps/rejected": -404.4368896484375,
"loss": 0.0038,
"rewards/accuracies": 0.9975000023841858,
"rewards/chosen": -2.470909357070923,
"rewards/margins": 12.393091201782227,
"rewards/rejected": -14.864001274108887,
"step": 6400
},
{
"epoch": 2.4941543257989087,
"eval_logits/chosen": -15.429072380065918,
"eval_logits/rejected": -15.39948844909668,
"eval_logps/chosen": -350.1909484863281,
"eval_logps/rejected": -391.09564208984375,
"eval_loss": 0.00028344389284029603,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.8532273769378662,
"eval_rewards/margins": 11.565349578857422,
"eval_rewards/rejected": -12.418577194213867,
"eval_runtime": 183.8395,
"eval_samples_per_second": 1.131,
"eval_steps_per_second": 0.566,
"step": 6400
},
{
"epoch": 2.533125487139517,
"grad_norm": 0.011729140765964985,
"learning_rate": 3.083860713002276e-05,
"logits/chosen": -19.238452911376953,
"logits/rejected": -18.347043991088867,
"logps/chosen": -370.990478515625,
"logps/rejected": -400.6825256347656,
"loss": 0.0039,
"rewards/accuracies": 0.9950000047683716,
"rewards/chosen": -2.3387069702148438,
"rewards/margins": 12.353742599487305,
"rewards/rejected": -14.692447662353516,
"step": 6500
},
{
"epoch": 2.572096648480125,
"grad_norm": 0.014655795879662037,
"learning_rate": 2.5992630039587377e-05,
"logits/chosen": -19.383378982543945,
"logits/rejected": -18.424985885620117,
"logps/chosen": -380.931396484375,
"logps/rejected": -399.5527648925781,
"loss": 0.004,
"rewards/accuracies": 0.9950000047683716,
"rewards/chosen": -2.246741771697998,
"rewards/margins": 12.052864074707031,
"rewards/rejected": -14.299607276916504,
"step": 6600
},
{
"epoch": 2.572096648480125,
"eval_logits/chosen": -15.413681983947754,
"eval_logits/rejected": -15.38673210144043,
"eval_logps/chosen": -350.96832275390625,
"eval_logps/rejected": -392.92138671875,
"eval_loss": 0.0002713745925575495,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.9309618473052979,
"eval_rewards/margins": 11.670186996459961,
"eval_rewards/rejected": -12.601149559020996,
"eval_runtime": 183.927,
"eval_samples_per_second": 1.131,
"eval_steps_per_second": 0.565,
"step": 6600
},
{
"epoch": 2.6110678098207325,
"grad_norm": 0.007622725795954466,
"learning_rate": 2.153984763949371e-05,
"logits/chosen": -19.36250114440918,
"logits/rejected": -18.28182029724121,
"logps/chosen": -383.64678955078125,
"logps/rejected": -392.4764404296875,
"loss": 0.0051,
"rewards/accuracies": 0.9950000047683716,
"rewards/chosen": -2.286069631576538,
"rewards/margins": 12.199385643005371,
"rewards/rejected": -14.485455513000488,
"step": 6700
},
{
"epoch": 2.6500389711613406,
"grad_norm": 0.01298923883587122,
"learning_rate": 1.7488075790430934e-05,
"logits/chosen": -19.342336654663086,
"logits/rejected": -18.265600204467773,
"logps/chosen": -386.0182800292969,
"logps/rejected": -406.8291015625,
"loss": 0.0025,
"rewards/accuracies": 0.9975000023841858,
"rewards/chosen": -2.360260009765625,
"rewards/margins": 12.328630447387695,
"rewards/rejected": -14.68889045715332,
"step": 6800
},
{
"epoch": 2.6500389711613406,
"eval_logits/chosen": -15.421009063720703,
"eval_logits/rejected": -15.389812469482422,
"eval_logps/chosen": -350.44403076171875,
"eval_logps/rejected": -392.24505615234375,
"eval_loss": 0.00027188131934963167,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.8785340785980225,
"eval_rewards/margins": 11.654979705810547,
"eval_rewards/rejected": -12.533516883850098,
"eval_runtime": 183.8793,
"eval_samples_per_second": 1.131,
"eval_steps_per_second": 0.566,
"step": 6800
},
{
"epoch": 2.6890101325019486,
"grad_norm": 0.001023565884679556,
"learning_rate": 1.3844426469017707e-05,
"logits/chosen": -19.42437171936035,
"logits/rejected": -18.330951690673828,
"logps/chosen": -389.0,
"logps/rejected": -406.2248840332031,
"loss": 0.0029,
"rewards/accuracies": 0.9962499737739563,
"rewards/chosen": -2.3574695587158203,
"rewards/margins": 12.130472183227539,
"rewards/rejected": -14.48794174194336,
"step": 6900
},
{
"epoch": 2.7279812938425563,
"grad_norm": 0.030132969841361046,
"learning_rate": 1.061529528432198e-05,
"logits/chosen": -19.441736221313477,
"logits/rejected": -18.430885314941406,
"logps/chosen": -375.5260009765625,
"logps/rejected": -398.24298095703125,
"loss": 0.0012,
"rewards/accuracies": 0.9987499713897705,
"rewards/chosen": -2.37859845161438,
"rewards/margins": 12.304988861083984,
"rewards/rejected": -14.683588981628418,
"step": 7000
},
{
"epoch": 2.7279812938425563,
"eval_logits/chosen": -15.412691116333008,
"eval_logits/rejected": -15.382089614868164,
"eval_logps/chosen": -351.04168701171875,
"eval_logps/rejected": -393.3232421875,
"eval_loss": 0.00028384948382154107,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.9383015036582947,
"eval_rewards/margins": 11.703031539916992,
"eval_rewards/rejected": -12.641332626342773,
"eval_runtime": 183.1659,
"eval_samples_per_second": 1.136,
"eval_steps_per_second": 0.568,
"step": 7000
},
{
"epoch": 2.7669524551831643,
"grad_norm": 0.29376310110092163,
"learning_rate": 7.806350251804484e-06,
"logits/chosen": -19.37668228149414,
"logits/rejected": -18.328815460205078,
"logps/chosen": -368.5742492675781,
"logps/rejected": -397.5633544921875,
"loss": 0.003,
"rewards/accuracies": 0.9962499737739563,
"rewards/chosen": -2.3045263290405273,
"rewards/margins": 12.480491638183594,
"rewards/rejected": -14.785019874572754,
"step": 7100
},
{
"epoch": 2.8059236165237724,
"grad_norm": 0.002321546198800206,
"learning_rate": 5.422521844388683e-06,
"logits/chosen": -19.359325408935547,
"logits/rejected": -18.383474349975586,
"logps/chosen": -373.9019470214844,
"logps/rejected": -399.3661193847656,
"loss": 0.0047,
"rewards/accuracies": 0.9937499761581421,
"rewards/chosen": -2.3699896335601807,
"rewards/margins": 12.351306915283203,
"rewards/rejected": -14.721295356750488,
"step": 7200
},
{
"epoch": 2.8059236165237724,
"eval_logits/chosen": -15.414888381958008,
"eval_logits/rejected": -15.384535789489746,
"eval_logps/chosen": -350.7855224609375,
"eval_logps/rejected": -393.17767333984375,
"eval_loss": 0.00028104818193241954,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.9126843214035034,
"eval_rewards/margins": 11.714097023010254,
"eval_rewards/rejected": -12.626781463623047,
"eval_runtime": 183.8851,
"eval_samples_per_second": 1.131,
"eval_steps_per_second": 0.566,
"step": 7200
},
{
"epoch": 2.8448947778643805,
"grad_norm": 0.027766738086938858,
"learning_rate": 3.4679943381216438e-06,
"logits/chosen": -19.325197219848633,
"logits/rejected": -18.378944396972656,
"logps/chosen": -379.8866271972656,
"logps/rejected": -401.4820251464844,
"loss": 0.0004,
"rewards/accuracies": 1.0,
"rewards/chosen": -2.4111533164978027,
"rewards/margins": 12.260860443115234,
"rewards/rejected": -14.672014236450195,
"step": 7300
},
{
"epoch": 2.8838659392049886,
"grad_norm": 0.0003760048421099782,
"learning_rate": 1.9461984676158727e-06,
"logits/chosen": -19.34538459777832,
"logits/rejected": -18.292407989501953,
"logps/chosen": -386.7870788574219,
"logps/rejected": -399.7149353027344,
"loss": 0.003,
"rewards/accuracies": 0.9975000023841858,
"rewards/chosen": -2.498373508453369,
"rewards/margins": 12.094259262084961,
"rewards/rejected": -14.592632293701172,
"step": 7400
},
{
"epoch": 2.8838659392049886,
"eval_logits/chosen": -15.413555145263672,
"eval_logits/rejected": -15.383076667785645,
"eval_logps/chosen": -350.8740539550781,
"eval_logps/rejected": -393.42462158203125,
"eval_loss": 0.0002813572355080396,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.9215376377105713,
"eval_rewards/margins": 11.729934692382812,
"eval_rewards/rejected": -12.651473045349121,
"eval_runtime": 184.2047,
"eval_samples_per_second": 1.129,
"eval_steps_per_second": 0.565,
"step": 7400
},
{
"epoch": 2.922837100545596,
"grad_norm": 0.0007408323581330478,
"learning_rate": 8.598054041644155e-07,
"logits/chosen": -19.447223663330078,
"logits/rejected": -18.43070411682129,
"logps/chosen": -382.068359375,
"logps/rejected": -402.309814453125,
"loss": 0.002,
"rewards/accuracies": 0.9987499713897705,
"rewards/chosen": -2.2234978675842285,
"rewards/margins": 12.309508323669434,
"rewards/rejected": -14.533007621765137,
"step": 7500
},
{
"epoch": 2.9618082618862043,
"grad_norm": 0.0027539017610251904,
"learning_rate": 2.107220670987675e-07,
"logits/chosen": -19.346935272216797,
"logits/rejected": -18.29763412475586,
"logps/chosen": -388.3053894042969,
"logps/rejected": -401.0611572265625,
"loss": 0.003,
"rewards/accuracies": 0.9975000023841858,
"rewards/chosen": -2.2870028018951416,
"rewards/margins": 12.200346946716309,
"rewards/rejected": -14.487349510192871,
"step": 7600
},
{
"epoch": 2.9618082618862043,
"eval_logits/chosen": -15.41310977935791,
"eval_logits/rejected": -15.382636070251465,
"eval_logps/chosen": -350.8951721191406,
"eval_logps/rejected": -393.48431396484375,
"eval_loss": 0.0002806605480145663,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": -0.9236502051353455,
"eval_rewards/margins": 11.733796119689941,
"eval_rewards/rejected": -12.657448768615723,
"eval_runtime": 183.4198,
"eval_samples_per_second": 1.134,
"eval_steps_per_second": 0.567,
"step": 7600
}
],
"logging_steps": 100,
"max_steps": 7698,
"num_input_tokens_seen": 0,
"num_train_epochs": 3,
"save_steps": 200,
"total_flos": 0.0,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}