ziansu's picture
Training in progress, step 550, checkpoint
34954b7 verified
raw
history blame
36.7 kB
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.6984126984126984,
"eval_steps": 50,
"global_step": 550,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.012698412698412698,
"grad_norm": 0.04716634005308151,
"learning_rate": 4.999451708687114e-06,
"logits/chosen": 14.883691787719727,
"logits/rejected": 15.016583442687988,
"logps/chosen": -0.29512909054756165,
"logps/rejected": -0.30033987760543823,
"loss": 1.0007,
"rewards/accuracies": 0.36250001192092896,
"rewards/chosen": -0.4426936209201813,
"rewards/margins": 0.007816222496330738,
"rewards/rejected": -0.45050984621047974,
"step": 10
},
{
"epoch": 0.025396825396825397,
"grad_norm": 0.04392225295305252,
"learning_rate": 4.997807075247147e-06,
"logits/chosen": 14.705224990844727,
"logits/rejected": 14.737253189086914,
"logps/chosen": -0.3201700747013092,
"logps/rejected": -0.280050128698349,
"loss": 0.9984,
"rewards/accuracies": 0.26249998807907104,
"rewards/chosen": -0.4802550673484802,
"rewards/margins": -0.060179851949214935,
"rewards/rejected": -0.4200752377510071,
"step": 20
},
{
"epoch": 0.0380952380952381,
"grad_norm": 0.04989476501941681,
"learning_rate": 4.9950668210706795e-06,
"logits/chosen": 15.147977828979492,
"logits/rejected": 15.080279350280762,
"logps/chosen": -0.3157380223274231,
"logps/rejected": -0.2997627556324005,
"loss": 0.9991,
"rewards/accuracies": 0.2874999940395355,
"rewards/chosen": -0.47360706329345703,
"rewards/margins": -0.023962898179888725,
"rewards/rejected": -0.44964417815208435,
"step": 30
},
{
"epoch": 0.050793650793650794,
"grad_norm": 0.054729390889406204,
"learning_rate": 4.9912321481237616e-06,
"logits/chosen": 15.301165580749512,
"logits/rejected": 15.276555061340332,
"logps/chosen": -0.30470195412635803,
"logps/rejected": -0.29767152667045593,
"loss": 0.9849,
"rewards/accuracies": 0.36250001192092896,
"rewards/chosen": -0.45705294609069824,
"rewards/margins": -0.010545584373176098,
"rewards/rejected": -0.44650736451148987,
"step": 40
},
{
"epoch": 0.06349206349206349,
"grad_norm": 0.06035450100898743,
"learning_rate": 4.986304738420684e-06,
"logits/chosen": 14.600168228149414,
"logits/rejected": 14.7944917678833,
"logps/chosen": -0.32278841733932495,
"logps/rejected": -0.3014402687549591,
"loss": 0.9991,
"rewards/accuracies": 0.3499999940395355,
"rewards/chosen": -0.48418259620666504,
"rewards/margins": -0.03202226758003235,
"rewards/rejected": -0.4521603584289551,
"step": 50
},
{
"epoch": 0.06349206349206349,
"eval_logits/chosen": 15.261337280273438,
"eval_logits/rejected": 15.51547908782959,
"eval_logps/chosen": -0.3022651970386505,
"eval_logps/rejected": -0.3061661124229431,
"eval_loss": 0.9846106171607971,
"eval_rewards/accuracies": 0.40625,
"eval_rewards/chosen": -0.4533977508544922,
"eval_rewards/margins": 0.005851435009390116,
"eval_rewards/rejected": -0.45924919843673706,
"eval_runtime": 18.4033,
"eval_samples_per_second": 27.712,
"eval_steps_per_second": 3.478,
"step": 50
},
{
"epoch": 0.0761904761904762,
"grad_norm": 0.052623528987169266,
"learning_rate": 4.980286753286196e-06,
"logits/chosen": 15.2726469039917,
"logits/rejected": 15.245884895324707,
"logps/chosen": -0.31394433975219727,
"logps/rejected": -0.2805258333683014,
"loss": 1.0063,
"rewards/accuracies": 0.22499999403953552,
"rewards/chosen": -0.4709165096282959,
"rewards/margins": -0.05012776702642441,
"rewards/rejected": -0.4207887053489685,
"step": 60
},
{
"epoch": 0.08888888888888889,
"grad_norm": 0.3512271046638489,
"learning_rate": 4.973180832407471e-06,
"logits/chosen": 15.302217483520508,
"logits/rejected": 15.387911796569824,
"logps/chosen": -0.30377697944641113,
"logps/rejected": -0.29276782274246216,
"loss": 0.9888,
"rewards/accuracies": 0.3499999940395355,
"rewards/chosen": -0.4556654393672943,
"rewards/margins": -0.016513748094439507,
"rewards/rejected": -0.43915170431137085,
"step": 70
},
{
"epoch": 0.10158730158730159,
"grad_norm": 0.056646134704351425,
"learning_rate": 4.964990092676263e-06,
"logits/chosen": 15.843823432922363,
"logits/rejected": 15.895855903625488,
"logps/chosen": -0.3091468811035156,
"logps/rejected": -0.27621737122535706,
"loss": 0.9926,
"rewards/accuracies": 0.3375000059604645,
"rewards/chosen": -0.46372032165527344,
"rewards/margins": -0.04939427971839905,
"rewards/rejected": -0.4143260419368744,
"step": 80
},
{
"epoch": 0.11428571428571428,
"grad_norm": 0.062348950654268265,
"learning_rate": 4.9557181268217225e-06,
"logits/chosen": 15.522564888000488,
"logits/rejected": 15.576242446899414,
"logps/chosen": -0.3092329502105713,
"logps/rejected": -0.27670228481292725,
"loss": 0.9812,
"rewards/accuracies": 0.22499999403953552,
"rewards/chosen": -0.4638494551181793,
"rewards/margins": -0.048796020448207855,
"rewards/rejected": -0.41505345702171326,
"step": 90
},
{
"epoch": 0.12698412698412698,
"grad_norm": 0.06779928505420685,
"learning_rate": 4.9453690018345144e-06,
"logits/chosen": 15.999185562133789,
"logits/rejected": 15.802484512329102,
"logps/chosen": -0.3173079490661621,
"logps/rejected": -0.283602774143219,
"loss": 0.981,
"rewards/accuracies": 0.2750000059604645,
"rewards/chosen": -0.4759618639945984,
"rewards/margins": -0.050557754933834076,
"rewards/rejected": -0.4254041314125061,
"step": 100
},
{
"epoch": 0.12698412698412698,
"eval_logits/chosen": 15.745854377746582,
"eval_logits/rejected": 16.0076961517334,
"eval_logps/chosen": -0.29866182804107666,
"eval_logps/rejected": -0.3121251165866852,
"eval_loss": 0.9764130115509033,
"eval_rewards/accuracies": 0.453125,
"eval_rewards/chosen": -0.4479926824569702,
"eval_rewards/margins": 0.02019493840634823,
"eval_rewards/rejected": -0.4681876599788666,
"eval_runtime": 18.3383,
"eval_samples_per_second": 27.811,
"eval_steps_per_second": 3.49,
"step": 100
},
{
"epoch": 0.13968253968253969,
"grad_norm": 0.07346926629543304,
"learning_rate": 4.933947257182901e-06,
"logits/chosen": 15.723541259765625,
"logits/rejected": 15.764880180358887,
"logps/chosen": -0.3212783634662628,
"logps/rejected": -0.3159862160682678,
"loss": 0.9859,
"rewards/accuracies": 0.4124999940395355,
"rewards/chosen": -0.4819175601005554,
"rewards/margins": -0.00793826300650835,
"rewards/rejected": -0.47397929430007935,
"step": 110
},
{
"epoch": 0.1523809523809524,
"grad_norm": 0.06748441606760025,
"learning_rate": 4.921457902821578e-06,
"logits/chosen": 15.817390441894531,
"logits/rejected": 16.023067474365234,
"logps/chosen": -0.29811763763427734,
"logps/rejected": -0.30654552578926086,
"loss": 0.9827,
"rewards/accuracies": 0.375,
"rewards/chosen": -0.447176456451416,
"rewards/margins": 0.01264181174337864,
"rewards/rejected": -0.4598182737827301,
"step": 120
},
{
"epoch": 0.16507936507936508,
"grad_norm": 0.08042758703231812,
"learning_rate": 4.907906416994146e-06,
"logits/chosen": 15.45033073425293,
"logits/rejected": 15.865328788757324,
"logps/chosen": -0.27259278297424316,
"logps/rejected": -0.31338179111480713,
"loss": 0.9743,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.40888920426368713,
"rewards/margins": 0.06118353083729744,
"rewards/rejected": -0.4700726866722107,
"step": 130
},
{
"epoch": 0.17777777777777778,
"grad_norm": 0.15669430792331696,
"learning_rate": 4.893298743830168e-06,
"logits/chosen": 16.17765235900879,
"logits/rejected": 15.951130867004395,
"logps/chosen": -0.3015133738517761,
"logps/rejected": -0.2970424294471741,
"loss": 0.9824,
"rewards/accuracies": 0.36250001192092896,
"rewards/chosen": -0.4522700905799866,
"rewards/margins": -0.006706444080919027,
"rewards/rejected": -0.44556355476379395,
"step": 140
},
{
"epoch": 0.19047619047619047,
"grad_norm": 0.07115967571735382,
"learning_rate": 4.8776412907378845e-06,
"logits/chosen": 15.552728652954102,
"logits/rejected": 15.788042068481445,
"logps/chosen": -0.28624147176742554,
"logps/rejected": -0.3222900331020355,
"loss": 0.9727,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.4293622076511383,
"rewards/margins": 0.05407290905714035,
"rewards/rejected": -0.48343515396118164,
"step": 150
},
{
"epoch": 0.19047619047619047,
"eval_logits/chosen": 15.72143268585205,
"eval_logits/rejected": 15.974818229675293,
"eval_logps/chosen": -0.29440543055534363,
"eval_logps/rejected": -0.33279579877853394,
"eval_loss": 0.9540849328041077,
"eval_rewards/accuracies": 0.546875,
"eval_rewards/chosen": -0.44160816073417664,
"eval_rewards/margins": 0.05758553743362427,
"eval_rewards/rejected": -0.4991936683654785,
"eval_runtime": 18.335,
"eval_samples_per_second": 27.816,
"eval_steps_per_second": 3.491,
"step": 150
},
{
"epoch": 0.20317460317460317,
"grad_norm": 0.1144075095653534,
"learning_rate": 4.860940925593703e-06,
"logits/chosen": 15.63622760772705,
"logits/rejected": 15.758941650390625,
"logps/chosen": -0.310846745967865,
"logps/rejected": -0.35185256600379944,
"loss": 0.9517,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.4662701487541199,
"rewards/margins": 0.061508744955062866,
"rewards/rejected": -0.5277789235115051,
"step": 160
},
{
"epoch": 0.21587301587301588,
"grad_norm": 0.1173442155122757,
"learning_rate": 4.84320497372973e-06,
"logits/chosen": 15.8660306930542,
"logits/rejected": 15.83642292022705,
"logps/chosen": -0.279310017824173,
"logps/rejected": -0.3301998972892761,
"loss": 0.9439,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.41896501183509827,
"rewards/margins": 0.07633484899997711,
"rewards/rejected": -0.49529990553855896,
"step": 170
},
{
"epoch": 0.22857142857142856,
"grad_norm": 0.204214945435524,
"learning_rate": 4.824441214720629e-06,
"logits/chosen": 15.557527542114258,
"logits/rejected": 15.321504592895508,
"logps/chosen": -0.3029389977455139,
"logps/rejected": -0.3163699507713318,
"loss": 0.9369,
"rewards/accuracies": 0.38749998807907104,
"rewards/chosen": -0.4544084668159485,
"rewards/margins": 0.020146463066339493,
"rewards/rejected": -0.47455495595932007,
"step": 180
},
{
"epoch": 0.24126984126984127,
"grad_norm": 0.14012843370437622,
"learning_rate": 4.804657878971252e-06,
"logits/chosen": 15.567469596862793,
"logits/rejected": 15.717155456542969,
"logps/chosen": -0.3400501310825348,
"logps/rejected": -0.3475819230079651,
"loss": 0.9443,
"rewards/accuracies": 0.38749998807907104,
"rewards/chosen": -0.5100752115249634,
"rewards/margins": 0.011297682300209999,
"rewards/rejected": -0.5213728547096252,
"step": 190
},
{
"epoch": 0.25396825396825395,
"grad_norm": 0.12908300757408142,
"learning_rate": 4.783863644106502e-06,
"logits/chosen": 14.985162734985352,
"logits/rejected": 15.223039627075195,
"logps/chosen": -0.2876646816730499,
"logps/rejected": -0.3777172267436981,
"loss": 0.9034,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.4314970076084137,
"rewards/margins": 0.13507887721061707,
"rewards/rejected": -0.5665758848190308,
"step": 200
},
{
"epoch": 0.25396825396825395,
"eval_logits/chosen": 15.242753028869629,
"eval_logits/rejected": 15.523069381713867,
"eval_logps/chosen": -0.2961229085922241,
"eval_logps/rejected": -0.3966684341430664,
"eval_loss": 0.9075753092765808,
"eval_rewards/accuracies": 0.59375,
"eval_rewards/chosen": -0.4441843330860138,
"eval_rewards/margins": 0.1508183479309082,
"eval_rewards/rejected": -0.5950026512145996,
"eval_runtime": 18.341,
"eval_samples_per_second": 27.807,
"eval_steps_per_second": 3.489,
"step": 200
},
{
"epoch": 0.26666666666666666,
"grad_norm": 0.13152211904525757,
"learning_rate": 4.762067631165049e-06,
"logits/chosen": 15.283185958862305,
"logits/rejected": 15.545951843261719,
"logps/chosen": -0.30045825242996216,
"logps/rejected": -0.4519944190979004,
"loss": 0.9085,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.45068734884262085,
"rewards/margins": 0.22730426490306854,
"rewards/rejected": -0.6779916286468506,
"step": 210
},
{
"epoch": 0.27936507936507937,
"grad_norm": 0.14594660699367523,
"learning_rate": 4.7392794005985324e-06,
"logits/chosen": 15.53101921081543,
"logits/rejected": 15.322232246398926,
"logps/chosen": -0.3336530327796936,
"logps/rejected": -0.36337023973464966,
"loss": 0.9188,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": -0.5004795789718628,
"rewards/margins": 0.04457578808069229,
"rewards/rejected": -0.5450553894042969,
"step": 220
},
{
"epoch": 0.2920634920634921,
"grad_norm": 0.14854153990745544,
"learning_rate": 4.715508948078037e-06,
"logits/chosen": 14.157003402709961,
"logits/rejected": 14.34089183807373,
"logps/chosen": -0.28324443101882935,
"logps/rejected": -0.41318759322166443,
"loss": 0.9015,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": -0.4248666763305664,
"rewards/margins": 0.19491472840309143,
"rewards/rejected": -0.6197813749313354,
"step": 230
},
{
"epoch": 0.3047619047619048,
"grad_norm": 0.19532087445259094,
"learning_rate": 4.690766700109659e-06,
"logits/chosen": 14.729398727416992,
"logits/rejected": 14.97203254699707,
"logps/chosen": -0.3013172149658203,
"logps/rejected": -0.41750937700271606,
"loss": 0.8814,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -0.45197582244873047,
"rewards/margins": 0.17428824305534363,
"rewards/rejected": -0.6262640357017517,
"step": 240
},
{
"epoch": 0.31746031746031744,
"grad_norm": 0.5795227885246277,
"learning_rate": 4.665063509461098e-06,
"logits/chosen": 14.328412055969238,
"logits/rejected": 14.645428657531738,
"logps/chosen": -0.3055194914340973,
"logps/rejected": -0.509621262550354,
"loss": 0.8478,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -0.45827922224998474,
"rewards/margins": 0.30615273118019104,
"rewards/rejected": -0.764431893825531,
"step": 250
},
{
"epoch": 0.31746031746031744,
"eval_logits/chosen": 14.310781478881836,
"eval_logits/rejected": 14.5652494430542,
"eval_logps/chosen": -0.3155084252357483,
"eval_logps/rejected": -0.8301784992218018,
"eval_loss": 0.7700436115264893,
"eval_rewards/accuracies": 0.640625,
"eval_rewards/chosen": -0.47326260805130005,
"eval_rewards/margins": 0.7720052003860474,
"eval_rewards/rejected": -1.2452678680419922,
"eval_runtime": 18.3413,
"eval_samples_per_second": 27.806,
"eval_steps_per_second": 3.489,
"step": 250
},
{
"epoch": 0.33015873015873015,
"grad_norm": 0.24876689910888672,
"learning_rate": 4.638410650401267e-06,
"logits/chosen": 14.115270614624023,
"logits/rejected": 14.167577743530273,
"logps/chosen": -0.32328444719314575,
"logps/rejected": -0.8564422726631165,
"loss": 0.795,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": -0.484926700592041,
"rewards/margins": 0.7997367978096008,
"rewards/rejected": -1.284663438796997,
"step": 260
},
{
"epoch": 0.34285714285714286,
"grad_norm": 0.5779634118080139,
"learning_rate": 4.610819813755038e-06,
"logits/chosen": 14.019407272338867,
"logits/rejected": 13.600751876831055,
"logps/chosen": -0.35225847363471985,
"logps/rejected": -1.1924464702606201,
"loss": 0.7644,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.5283876657485962,
"rewards/margins": 1.2602821588516235,
"rewards/rejected": -1.7886695861816406,
"step": 270
},
{
"epoch": 0.35555555555555557,
"grad_norm": 0.2085695117712021,
"learning_rate": 4.582303101775249e-06,
"logits/chosen": 13.448820114135742,
"logits/rejected": 13.464624404907227,
"logps/chosen": -0.3442023992538452,
"logps/rejected": -1.4783053398132324,
"loss": 0.7671,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.516303539276123,
"rewards/margins": 1.7011544704437256,
"rewards/rejected": -2.2174580097198486,
"step": 280
},
{
"epoch": 0.3682539682539683,
"grad_norm": 0.2825860381126404,
"learning_rate": 4.55287302283426e-06,
"logits/chosen": 13.038851737976074,
"logits/rejected": 13.013893127441406,
"logps/chosen": -0.3524690568447113,
"logps/rejected": -1.2354861497879028,
"loss": 0.7922,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.5287035703659058,
"rewards/margins": 1.3245255947113037,
"rewards/rejected": -1.8532291650772095,
"step": 290
},
{
"epoch": 0.38095238095238093,
"grad_norm": 0.30771639943122864,
"learning_rate": 4.522542485937369e-06,
"logits/chosen": 13.67170524597168,
"logits/rejected": 13.185449600219727,
"logps/chosen": -0.3915162682533264,
"logps/rejected": -1.3473880290985107,
"loss": 0.7506,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.587274432182312,
"rewards/margins": 1.433807611465454,
"rewards/rejected": -2.0210821628570557,
"step": 300
},
{
"epoch": 0.38095238095238093,
"eval_logits/chosen": 13.198552131652832,
"eval_logits/rejected": 13.458879470825195,
"eval_logps/chosen": -0.37132135033607483,
"eval_logps/rejected": -1.4595866203308105,
"eval_loss": 0.7391816973686218,
"eval_rewards/accuracies": 0.640625,
"eval_rewards/chosen": -0.5569820404052734,
"eval_rewards/margins": 1.6323981285095215,
"eval_rewards/rejected": -2.189380168914795,
"eval_runtime": 18.3117,
"eval_samples_per_second": 27.851,
"eval_steps_per_second": 3.495,
"step": 300
},
{
"epoch": 0.39365079365079364,
"grad_norm": 0.3021286129951477,
"learning_rate": 4.491324795060491e-06,
"logits/chosen": 13.347516059875488,
"logits/rejected": 13.285726547241211,
"logps/chosen": -0.4108191132545471,
"logps/rejected": -1.436702013015747,
"loss": 0.7134,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.6162286400794983,
"rewards/margins": 1.5388243198394775,
"rewards/rejected": -2.15505313873291,
"step": 310
},
{
"epoch": 0.40634920634920635,
"grad_norm": 0.40123897790908813,
"learning_rate": 4.4592336433146e-06,
"logits/chosen": 12.949071884155273,
"logits/rejected": 12.86626148223877,
"logps/chosen": -0.42898029088974,
"logps/rejected": -1.1741807460784912,
"loss": 0.74,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": -0.6434704065322876,
"rewards/margins": 1.1178009510040283,
"rewards/rejected": -1.7612712383270264,
"step": 320
},
{
"epoch": 0.41904761904761906,
"grad_norm": 0.4613189697265625,
"learning_rate": 4.426283106939474e-06,
"logits/chosen": 12.796838760375977,
"logits/rejected": 12.524165153503418,
"logps/chosen": -0.48975634574890137,
"logps/rejected": -1.2766364812850952,
"loss": 0.7408,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.7346345782279968,
"rewards/margins": 1.180320143699646,
"rewards/rejected": -1.9149547815322876,
"step": 330
},
{
"epoch": 0.43174603174603177,
"grad_norm": 0.49141305685043335,
"learning_rate": 4.3924876391293915e-06,
"logits/chosen": 12.58080005645752,
"logits/rejected": 12.5891695022583,
"logps/chosen": -0.5303409099578857,
"logps/rejected": -1.6262977123260498,
"loss": 0.7501,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.7955113649368286,
"rewards/margins": 1.643935203552246,
"rewards/rejected": -2.4394466876983643,
"step": 340
},
{
"epoch": 0.4444444444444444,
"grad_norm": 0.7533912062644958,
"learning_rate": 4.357862063693486e-06,
"logits/chosen": 12.15321159362793,
"logits/rejected": 12.38386344909668,
"logps/chosen": -0.5782582759857178,
"logps/rejected": -1.580635905265808,
"loss": 0.7405,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.8673874139785767,
"rewards/margins": 1.5035666227340698,
"rewards/rejected": -2.3709540367126465,
"step": 350
},
{
"epoch": 0.4444444444444444,
"eval_logits/chosen": 12.111034393310547,
"eval_logits/rejected": 12.33246898651123,
"eval_logps/chosen": -0.5941734910011292,
"eval_logps/rejected": -1.6774102449417114,
"eval_loss": 0.6953701972961426,
"eval_rewards/accuracies": 0.765625,
"eval_rewards/chosen": -0.8912601470947266,
"eval_rewards/margins": 1.6248550415039062,
"eval_rewards/rejected": -2.516115188598633,
"eval_runtime": 18.3366,
"eval_samples_per_second": 27.813,
"eval_steps_per_second": 3.49,
"step": 350
},
{
"epoch": 0.45714285714285713,
"grad_norm": 1.168564796447754,
"learning_rate": 4.322421568553529e-06,
"logits/chosen": 11.706710815429688,
"logits/rejected": 11.451011657714844,
"logps/chosen": -0.758882462978363,
"logps/rejected": -1.842167615890503,
"loss": 0.702,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -1.1383236646652222,
"rewards/margins": 1.6249277591705322,
"rewards/rejected": -2.763251304626465,
"step": 360
},
{
"epoch": 0.46984126984126984,
"grad_norm": 1.8962547779083252,
"learning_rate": 4.286181699082008e-06,
"logits/chosen": 11.41061019897461,
"logits/rejected": 11.534398078918457,
"logps/chosen": -1.1785552501678467,
"logps/rejected": -2.235652446746826,
"loss": 0.6781,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -1.7678327560424805,
"rewards/margins": 1.5856454372406006,
"rewards/rejected": -3.3534786701202393,
"step": 370
},
{
"epoch": 0.48253968253968255,
"grad_norm": 2.9152028560638428,
"learning_rate": 4.249158351283414e-06,
"logits/chosen": 11.582561492919922,
"logits/rejected": 11.439531326293945,
"logps/chosen": -1.9169971942901611,
"logps/rejected": -2.788179397583008,
"loss": 0.6033,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -2.875495672225952,
"rewards/margins": 1.306773066520691,
"rewards/rejected": -4.182269096374512,
"step": 380
},
{
"epoch": 0.49523809523809526,
"grad_norm": 1.5070641040802002,
"learning_rate": 4.211367764821722e-06,
"logits/chosen": 11.340807914733887,
"logits/rejected": 10.927321434020996,
"logps/chosen": -2.1075730323791504,
"logps/rejected": -3.4216556549072266,
"loss": 0.5698,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -3.1613595485687256,
"rewards/margins": 1.9711239337921143,
"rewards/rejected": -5.13248348236084,
"step": 390
},
{
"epoch": 0.5079365079365079,
"grad_norm": 2.6913516521453857,
"learning_rate": 4.172826515897146e-06,
"logits/chosen": 10.273612976074219,
"logits/rejected": 10.116204261779785,
"logps/chosen": -3.0047056674957275,
"logps/rejected": -4.053646564483643,
"loss": 0.5869,
"rewards/accuracies": 0.875,
"rewards/chosen": -4.507058143615723,
"rewards/margins": 1.5734113454818726,
"rewards/rejected": -6.080469608306885,
"step": 400
},
{
"epoch": 0.5079365079365079,
"eval_logits/chosen": 9.58273696899414,
"eval_logits/rejected": 9.765192031860352,
"eval_logps/chosen": -2.4895575046539307,
"eval_logps/rejected": -3.8314545154571533,
"eval_loss": 0.5232856273651123,
"eval_rewards/accuracies": 0.875,
"eval_rewards/chosen": -3.7343361377716064,
"eval_rewards/margins": 2.012845754623413,
"eval_rewards/rejected": -5.7471818923950195,
"eval_runtime": 18.3437,
"eval_samples_per_second": 27.802,
"eval_steps_per_second": 3.489,
"step": 400
},
{
"epoch": 0.5206349206349207,
"grad_norm": 3.7197885513305664,
"learning_rate": 4.133551509975264e-06,
"logits/chosen": 9.081324577331543,
"logits/rejected": 9.309822082519531,
"logps/chosen": -2.2554564476013184,
"logps/rejected": -3.2624363899230957,
"loss": 0.5774,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": -3.3831849098205566,
"rewards/margins": 1.5104694366455078,
"rewards/rejected": -4.893653869628906,
"step": 410
},
{
"epoch": 0.5333333333333333,
"grad_norm": 2.757873773574829,
"learning_rate": 4.093559974371725e-06,
"logits/chosen": 8.145927429199219,
"logits/rejected": 8.402036666870117,
"logps/chosen": -3.1110777854919434,
"logps/rejected": -4.202320098876953,
"loss": 0.5084,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": -4.666616916656494,
"rewards/margins": 1.636863350868225,
"rewards/rejected": -6.30348014831543,
"step": 420
},
{
"epoch": 0.546031746031746,
"grad_norm": 3.3764102458953857,
"learning_rate": 4.052869450695776e-06,
"logits/chosen": 8.132574081420898,
"logits/rejected": 8.386279106140137,
"logps/chosen": -3.0871124267578125,
"logps/rejected": -4.399088382720947,
"loss": 0.5233,
"rewards/accuracies": 0.875,
"rewards/chosen": -4.630668640136719,
"rewards/margins": 1.967963457107544,
"rewards/rejected": -6.598631858825684,
"step": 430
},
{
"epoch": 0.5587301587301587,
"grad_norm": 1.8188632726669312,
"learning_rate": 4.011497787155938e-06,
"logits/chosen": 8.104793548583984,
"logits/rejected": 8.25890064239502,
"logps/chosen": -3.598665237426758,
"logps/rejected": -5.2809929847717285,
"loss": 0.5272,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -5.397997856140137,
"rewards/margins": 2.5234923362731934,
"rewards/rejected": -7.921489715576172,
"step": 440
},
{
"epoch": 0.5714285714285714,
"grad_norm": 2.528883934020996,
"learning_rate": 3.969463130731183e-06,
"logits/chosen": 8.675338745117188,
"logits/rejected": 8.550267219543457,
"logps/chosen": -3.582223892211914,
"logps/rejected": -5.077345848083496,
"loss": 0.5717,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -5.373335838317871,
"rewards/margins": 2.2426836490631104,
"rewards/rejected": -7.616019248962402,
"step": 450
},
{
"epoch": 0.5714285714285714,
"eval_logits/chosen": 8.125189781188965,
"eval_logits/rejected": 8.239119529724121,
"eval_logps/chosen": -3.107839584350586,
"eval_logps/rejected": -4.726868629455566,
"eval_loss": 0.4568406939506531,
"eval_rewards/accuracies": 0.875,
"eval_rewards/chosen": -4.661759376525879,
"eval_rewards/margins": 2.4285435676574707,
"eval_rewards/rejected": -7.090303421020508,
"eval_runtime": 18.3222,
"eval_samples_per_second": 27.835,
"eval_steps_per_second": 3.493,
"step": 450
},
{
"epoch": 0.5841269841269842,
"grad_norm": 2.5079057216644287,
"learning_rate": 3.92678391921108e-06,
"logits/chosen": 8.503251075744629,
"logits/rejected": 8.457250595092773,
"logps/chosen": -3.499116897583008,
"logps/rejected": -5.12787389755249,
"loss": 0.4803,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": -5.2486748695373535,
"rewards/margins": 2.4431352615356445,
"rewards/rejected": -7.69180965423584,
"step": 460
},
{
"epoch": 0.5968253968253968,
"grad_norm": 1.869667887687683,
"learning_rate": 3.88347887310836e-06,
"logits/chosen": 7.972044467926025,
"logits/rejected": 7.825163841247559,
"logps/chosen": -3.6900856494903564,
"logps/rejected": -5.166090488433838,
"loss": 0.4733,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": -5.535128593444824,
"rewards/margins": 2.2140071392059326,
"rewards/rejected": -7.749135494232178,
"step": 470
},
{
"epoch": 0.6095238095238096,
"grad_norm": 2.8777735233306885,
"learning_rate": 3.839566987447492e-06,
"logits/chosen": 8.073511123657227,
"logits/rejected": 8.345193862915039,
"logps/chosen": -3.7006676197052,
"logps/rejected": -5.271113395690918,
"loss": 0.5048,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": -5.551001071929932,
"rewards/margins": 2.3556694984436035,
"rewards/rejected": -7.906670570373535,
"step": 480
},
{
"epoch": 0.6222222222222222,
"grad_norm": 2.7042839527130127,
"learning_rate": 3.795067523432826e-06,
"logits/chosen": 8.602261543273926,
"logits/rejected": 8.32430362701416,
"logps/chosen": -3.9612879753112793,
"logps/rejected": -6.123431205749512,
"loss": 0.4728,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -5.94193172454834,
"rewards/margins": 3.2432150840759277,
"rewards/rejected": -9.185147285461426,
"step": 490
},
{
"epoch": 0.6349206349206349,
"grad_norm": 2.1625821590423584,
"learning_rate": 3.7500000000000005e-06,
"logits/chosen": 7.857954978942871,
"logits/rejected": 7.460604190826416,
"logps/chosen": -3.581023693084717,
"logps/rejected": -5.477323532104492,
"loss": 0.4296,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": -5.371535778045654,
"rewards/margins": 2.844449520111084,
"rewards/rejected": -8.215986251831055,
"step": 500
},
{
"epoch": 0.6349206349206349,
"eval_logits/chosen": 7.520671844482422,
"eval_logits/rejected": 7.586723327636719,
"eval_logps/chosen": -3.1659929752349854,
"eval_logps/rejected": -4.955198764801025,
"eval_loss": 0.4266711175441742,
"eval_rewards/accuracies": 0.890625,
"eval_rewards/chosen": -4.748989105224609,
"eval_rewards/margins": 2.683809518814087,
"eval_rewards/rejected": -7.432799339294434,
"eval_runtime": 18.3456,
"eval_samples_per_second": 27.8,
"eval_steps_per_second": 3.489,
"step": 500
},
{
"epoch": 0.6476190476190476,
"grad_norm": 4.2689104080200195,
"learning_rate": 3.7043841852542884e-06,
"logits/chosen": 7.948687553405762,
"logits/rejected": 7.656899929046631,
"logps/chosen": -2.8974719047546387,
"logps/rejected": -4.501969337463379,
"loss": 0.4981,
"rewards/accuracies": 0.875,
"rewards/chosen": -4.346207618713379,
"rewards/margins": 2.4067459106445312,
"rewards/rejected": -6.752954006195068,
"step": 510
},
{
"epoch": 0.6603174603174603,
"grad_norm": 3.299290895462036,
"learning_rate": 3.658240087799655e-06,
"logits/chosen": 7.676476955413818,
"logits/rejected": 7.332755088806152,
"logps/chosen": -3.9289379119873047,
"logps/rejected": -5.498073577880859,
"loss": 0.4736,
"rewards/accuracies": 0.8374999761581421,
"rewards/chosen": -5.893406867980957,
"rewards/margins": 2.353703498840332,
"rewards/rejected": -8.247110366821289,
"step": 520
},
{
"epoch": 0.6730158730158731,
"grad_norm": 3.1058828830718994,
"learning_rate": 3.611587947962319e-06,
"logits/chosen": 7.516615867614746,
"logits/rejected": 7.344383239746094,
"logps/chosen": -3.57861328125,
"logps/rejected": -5.48795223236084,
"loss": 0.502,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": -5.367920398712158,
"rewards/margins": 2.8640081882476807,
"rewards/rejected": -8.231928825378418,
"step": 530
},
{
"epoch": 0.6857142857142857,
"grad_norm": 3.874868869781494,
"learning_rate": 3.564448228912682e-06,
"logits/chosen": 7.619947910308838,
"logits/rejected": 7.048402309417725,
"logps/chosen": -4.164804935455322,
"logps/rejected": -5.771735191345215,
"loss": 0.44,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -6.247206687927246,
"rewards/margins": 2.4103951454162598,
"rewards/rejected": -8.657602310180664,
"step": 540
},
{
"epoch": 0.6984126984126984,
"grad_norm": 1.7583000659942627,
"learning_rate": 3.516841607689501e-06,
"logits/chosen": 7.068112850189209,
"logits/rejected": 7.042977333068848,
"logps/chosen": -3.760105609893799,
"logps/rejected": -5.3382248878479,
"loss": 0.4751,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": -5.640158653259277,
"rewards/margins": 2.3671793937683105,
"rewards/rejected": -8.00733757019043,
"step": 550
},
{
"epoch": 0.6984126984126984,
"eval_logits/chosen": 7.213573455810547,
"eval_logits/rejected": 7.25667667388916,
"eval_logps/chosen": -3.8712658882141113,
"eval_logps/rejected": -5.882909774780273,
"eval_loss": 0.3922204077243805,
"eval_rewards/accuracies": 0.90625,
"eval_rewards/chosen": -5.806899070739746,
"eval_rewards/margins": 3.017465591430664,
"eval_rewards/rejected": -8.824363708496094,
"eval_runtime": 18.3231,
"eval_samples_per_second": 27.834,
"eval_steps_per_second": 3.493,
"step": 550
}
],
"logging_steps": 10,
"max_steps": 1500,
"num_input_tokens_seen": 0,
"num_train_epochs": 2,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 1.3248943036836086e+18,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}