diff --git "a/checkpoint-1500/trainer_state.json" "b/checkpoint-1500/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-1500/trainer_state.json" @@ -0,0 +1,2763 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0869801576515357, + "eval_steps": 50, + "global_step": 1500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.007248346470961312, + "grad_norm": 0.07201674580574036, + "learning_rate": 4.999451708687114e-06, + "logits/chosen": -1.9793262481689453, + "logits/rejected": -2.5381760597229004, + "logps/chosen": -0.28126341104507446, + "logps/rejected": -0.3779803514480591, + "loss": 7.3904, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.4218950867652893, + "rewards/margins": 0.14507544040679932, + "rewards/rejected": -0.5669704675674438, + "step": 10 + }, + { + "epoch": 0.014496692941922623, + "grad_norm": 0.07562297582626343, + "learning_rate": 4.997807075247147e-06, + "logits/chosen": -2.0567643642425537, + "logits/rejected": -2.4989147186279297, + "logps/chosen": -0.27690139412879944, + "logps/rejected": -0.33544114232063293, + "loss": 7.3756, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.41535210609436035, + "rewards/margins": 0.08780960738658905, + "rewards/rejected": -0.5031617283821106, + "step": 20 + }, + { + "epoch": 0.021745039412883936, + "grad_norm": 0.09685570746660233, + "learning_rate": 4.9950668210706795e-06, + "logits/chosen": -2.10174298286438, + "logits/rejected": -2.378197431564331, + "logps/chosen": -0.26717427372932434, + "logps/rejected": -0.30565372109413147, + "loss": 7.451, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4007614254951477, + "rewards/margins": 0.05771917849779129, + "rewards/rejected": -0.4584805369377136, + "step": 30 + }, + { + "epoch": 0.028993385883845247, + "grad_norm": 0.08213861286640167, + "learning_rate": 4.9912321481237616e-06, + "logits/chosen": -2.1633317470550537, + "logits/rejected": -2.387866497039795, + "logps/chosen": -0.27634260058403015, + "logps/rejected": -0.37035584449768066, + "loss": 7.3892, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.4145139157772064, + "rewards/margins": 0.14101983606815338, + "rewards/rejected": -0.555533766746521, + "step": 40 + }, + { + "epoch": 0.03624173235480656, + "grad_norm": 0.08846044540405273, + "learning_rate": 4.986304738420684e-06, + "logits/chosen": -2.1402599811553955, + "logits/rejected": -2.4459526538848877, + "logps/chosen": -0.2535383999347687, + "logps/rejected": -0.3090876042842865, + "loss": 7.5171, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -0.3803076148033142, + "rewards/margins": 0.08332376182079315, + "rewards/rejected": -0.4636313319206238, + "step": 50 + }, + { + "epoch": 0.03624173235480656, + "eval_logits/chosen": -2.1165692806243896, + "eval_logits/rejected": -2.476428747177124, + "eval_logps/chosen": -0.2828062176704407, + "eval_logps/rejected": -0.3432886600494385, + "eval_loss": 0.9120001792907715, + "eval_rewards/accuracies": 0.5089285969734192, + "eval_rewards/chosen": -0.4242093861103058, + "eval_rewards/margins": 0.09072363376617432, + "eval_rewards/rejected": -0.5149329900741577, + "eval_runtime": 30.971, + "eval_samples_per_second": 28.801, + "eval_steps_per_second": 3.616, + "step": 50 + }, + { + "epoch": 0.04349007882576787, + "grad_norm": 0.11753705143928528, + "learning_rate": 4.980286753286196e-06, + "logits/chosen": -2.0575368404388428, + "logits/rejected": -2.5456700325012207, + "logps/chosen": -0.24622221291065216, + "logps/rejected": -0.32402220368385315, + "loss": 7.3926, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.36933332681655884, + "rewards/margins": 0.11669999361038208, + "rewards/rejected": -0.48603329062461853, + "step": 60 + }, + { + "epoch": 0.05073842529672919, + "grad_norm": 0.09996571391820908, + "learning_rate": 4.973180832407471e-06, + "logits/chosen": -1.9278606176376343, + "logits/rejected": -2.4620182514190674, + "logps/chosen": -0.2596542239189148, + "logps/rejected": -0.3665880560874939, + "loss": 7.185, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.3894812762737274, + "rewards/margins": 0.160400852560997, + "rewards/rejected": -0.5498821139335632, + "step": 70 + }, + { + "epoch": 0.057986771767690494, + "grad_norm": 0.07287321239709854, + "learning_rate": 4.964990092676263e-06, + "logits/chosen": -2.078031063079834, + "logits/rejected": -2.461479663848877, + "logps/chosen": -0.24513795971870422, + "logps/rejected": -0.3448730707168579, + "loss": 7.365, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.36770695447921753, + "rewards/margins": 0.14960262179374695, + "rewards/rejected": -0.5173095464706421, + "step": 80 + }, + { + "epoch": 0.0652351182386518, + "grad_norm": 0.09656044095754623, + "learning_rate": 4.9557181268217225e-06, + "logits/chosen": -1.9979238510131836, + "logits/rejected": -2.4381277561187744, + "logps/chosen": -0.24047240614891052, + "logps/rejected": -0.3277527987957001, + "loss": 7.2664, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.36070865392684937, + "rewards/margins": 0.13092057406902313, + "rewards/rejected": -0.4916292130947113, + "step": 90 + }, + { + "epoch": 0.07248346470961312, + "grad_norm": 0.08125138282775879, + "learning_rate": 4.9453690018345144e-06, + "logits/chosen": -1.8948112726211548, + "logits/rejected": -2.4755520820617676, + "logps/chosen": -0.20189261436462402, + "logps/rejected": -0.29732149839401245, + "loss": 7.2542, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.30283889174461365, + "rewards/margins": 0.14314329624176025, + "rewards/rejected": -0.44598227739334106, + "step": 100 + }, + { + "epoch": 0.07248346470961312, + "eval_logits/chosen": -2.12322735786438, + "eval_logits/rejected": -2.481174945831299, + "eval_logps/chosen": -0.2438412606716156, + "eval_logps/rejected": -0.3260033428668976, + "eval_loss": 0.891861081123352, + "eval_rewards/accuracies": 0.5625, + "eval_rewards/chosen": -0.3657619059085846, + "eval_rewards/margins": 0.12324309349060059, + "eval_rewards/rejected": -0.48900502920150757, + "eval_runtime": 30.3299, + "eval_samples_per_second": 29.41, + "eval_steps_per_second": 3.693, + "step": 100 + }, + { + "epoch": 0.07973181118057443, + "grad_norm": 0.05962231010198593, + "learning_rate": 4.933947257182901e-06, + "logits/chosen": -2.1134355068206787, + "logits/rejected": -2.516538381576538, + "logps/chosen": -0.237023264169693, + "logps/rejected": -0.31476154923439026, + "loss": 7.1749, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.3555349111557007, + "rewards/margins": 0.11660744249820709, + "rewards/rejected": -0.4721423089504242, + "step": 110 + }, + { + "epoch": 0.08698015765153574, + "grad_norm": 0.06015922501683235, + "learning_rate": 4.921457902821578e-06, + "logits/chosen": -2.0215041637420654, + "logits/rejected": -2.4902031421661377, + "logps/chosen": -0.1890055537223816, + "logps/rejected": -0.3192065358161926, + "loss": 7.142, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.2835083603858948, + "rewards/margins": 0.19530144333839417, + "rewards/rejected": -0.47880974411964417, + "step": 120 + }, + { + "epoch": 0.09422850412249706, + "grad_norm": 0.06430571526288986, + "learning_rate": 4.907906416994146e-06, + "logits/chosen": -2.0684752464294434, + "logits/rejected": -2.510018825531006, + "logps/chosen": -0.2073744535446167, + "logps/rejected": -0.3121300935745239, + "loss": 7.1438, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.31106168031692505, + "rewards/margins": 0.15713343024253845, + "rewards/rejected": -0.4681951403617859, + "step": 130 + }, + { + "epoch": 0.10147685059345837, + "grad_norm": 0.08829955011606216, + "learning_rate": 4.893298743830168e-06, + "logits/chosen": -2.026458263397217, + "logits/rejected": -2.496157646179199, + "logps/chosen": -0.19946983456611633, + "logps/rejected": -0.32050156593322754, + "loss": 7.1196, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2992047667503357, + "rewards/margins": 0.18154758214950562, + "rewards/rejected": -0.4807523787021637, + "step": 140 + }, + { + "epoch": 0.10872519706441967, + "grad_norm": 0.09773921221494675, + "learning_rate": 4.8776412907378845e-06, + "logits/chosen": -2.101029872894287, + "logits/rejected": -2.5849032402038574, + "logps/chosen": -0.18889756500720978, + "logps/rejected": -0.36427801847457886, + "loss": 7.1227, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.28334635496139526, + "rewards/margins": 0.26307064294815063, + "rewards/rejected": -0.5464169979095459, + "step": 150 + }, + { + "epoch": 0.10872519706441967, + "eval_logits/chosen": -2.1945455074310303, + "eval_logits/rejected": -2.559415578842163, + "eval_logps/chosen": -0.22209034860134125, + "eval_logps/rejected": -0.32476040720939636, + "eval_loss": 0.8760393261909485, + "eval_rewards/accuracies": 0.5803571343421936, + "eval_rewards/chosen": -0.33313554525375366, + "eval_rewards/margins": 0.15400508046150208, + "eval_rewards/rejected": -0.48714062571525574, + "eval_runtime": 30.3484, + "eval_samples_per_second": 29.392, + "eval_steps_per_second": 3.69, + "step": 150 + }, + { + "epoch": 0.11597354353538099, + "grad_norm": 0.09142427891492844, + "learning_rate": 4.860940925593703e-06, + "logits/chosen": -2.235069751739502, + "logits/rejected": -2.5513346195220947, + "logps/chosen": -0.19598741829395294, + "logps/rejected": -0.2772120535373688, + "loss": 7.0272, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -0.2939811050891876, + "rewards/margins": 0.12183700501918793, + "rewards/rejected": -0.41581812500953674, + "step": 160 + }, + { + "epoch": 0.1232218900063423, + "grad_norm": 0.11735275387763977, + "learning_rate": 4.84320497372973e-06, + "logits/chosen": -2.154263496398926, + "logits/rejected": -2.512010335922241, + "logps/chosen": -0.1798369437456131, + "logps/rejected": -0.30910637974739075, + "loss": 7.1603, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.26975542306900024, + "rewards/margins": 0.19390416145324707, + "rewards/rejected": -0.4636595845222473, + "step": 170 + }, + { + "epoch": 0.1304702364773036, + "grad_norm": 0.09398588538169861, + "learning_rate": 4.824441214720629e-06, + "logits/chosen": -2.292147159576416, + "logits/rejected": -2.581425189971924, + "logps/chosen": -0.18498703837394714, + "logps/rejected": -0.31237050890922546, + "loss": 6.995, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.2774805426597595, + "rewards/margins": 0.19107523560523987, + "rewards/rejected": -0.4685557782649994, + "step": 180 + }, + { + "epoch": 0.13771858294826492, + "grad_norm": 0.13551996648311615, + "learning_rate": 4.804657878971252e-06, + "logits/chosen": -2.274120330810547, + "logits/rejected": -2.5797386169433594, + "logps/chosen": -0.21168990433216095, + "logps/rejected": -0.34927254915237427, + "loss": 6.9941, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.317534863948822, + "rewards/margins": 0.20637397468090057, + "rewards/rejected": -0.5239088535308838, + "step": 190 + }, + { + "epoch": 0.14496692941922623, + "grad_norm": 0.15515944361686707, + "learning_rate": 4.783863644106502e-06, + "logits/chosen": -2.2116379737854004, + "logits/rejected": -2.6693203449249268, + "logps/chosen": -0.1971816122531891, + "logps/rejected": -0.3498842120170593, + "loss": 7.0413, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2957724332809448, + "rewards/margins": 0.22905388474464417, + "rewards/rejected": -0.5248263478279114, + "step": 200 + }, + { + "epoch": 0.14496692941922623, + "eval_logits/chosen": -2.3369696140289307, + "eval_logits/rejected": -2.7298672199249268, + "eval_logps/chosen": -0.22749511897563934, + "eval_logps/rejected": -0.3613782525062561, + "eval_loss": 0.8543878793716431, + "eval_rewards/accuracies": 0.5714285969734192, + "eval_rewards/chosen": -0.34124264121055603, + "eval_rewards/margins": 0.2008247673511505, + "eval_rewards/rejected": -0.5420674681663513, + "eval_runtime": 30.3397, + "eval_samples_per_second": 29.4, + "eval_steps_per_second": 3.692, + "step": 200 + }, + { + "epoch": 0.15221527589018755, + "grad_norm": 0.13039635121822357, + "learning_rate": 4.762067631165049e-06, + "logits/chosen": -2.2575619220733643, + "logits/rejected": -2.755174160003662, + "logps/chosen": -0.199218288064003, + "logps/rejected": -0.33260637521743774, + "loss": 6.882, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.2988274097442627, + "rewards/margins": 0.20008206367492676, + "rewards/rejected": -0.49890947341918945, + "step": 210 + }, + { + "epoch": 0.15946362236114886, + "grad_norm": 0.15020275115966797, + "learning_rate": 4.7392794005985324e-06, + "logits/chosen": -2.2949752807617188, + "logits/rejected": -2.7288482189178467, + "logps/chosen": -0.21987763047218323, + "logps/rejected": -0.385539174079895, + "loss": 6.9158, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.3298164904117584, + "rewards/margins": 0.24849233031272888, + "rewards/rejected": -0.5783087611198425, + "step": 220 + }, + { + "epoch": 0.16671196883211017, + "grad_norm": 0.16104522347450256, + "learning_rate": 4.715508948078037e-06, + "logits/chosen": -2.1837964057922363, + "logits/rejected": -2.813563585281372, + "logps/chosen": -0.2069036215543747, + "logps/rejected": -0.3996518552303314, + "loss": 6.8503, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.31035539507865906, + "rewards/margins": 0.28912240266799927, + "rewards/rejected": -0.5994777679443359, + "step": 230 + }, + { + "epoch": 0.1739603153030715, + "grad_norm": 0.16533692181110382, + "learning_rate": 4.690766700109659e-06, + "logits/chosen": -2.314570188522339, + "logits/rejected": -2.8867621421813965, + "logps/chosen": -0.18964803218841553, + "logps/rejected": -0.3997463583946228, + "loss": 6.667, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2844720482826233, + "rewards/margins": 0.3151474893093109, + "rewards/rejected": -0.5996195077896118, + "step": 240 + }, + { + "epoch": 0.1812086617740328, + "grad_norm": 0.24991220235824585, + "learning_rate": 4.665063509461098e-06, + "logits/chosen": -2.46189546585083, + "logits/rejected": -2.847446918487549, + "logps/chosen": -0.24189543724060059, + "logps/rejected": -0.4183991551399231, + "loss": 6.7146, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.3628431558609009, + "rewards/margins": 0.26475557684898376, + "rewards/rejected": -0.6275987029075623, + "step": 250 + }, + { + "epoch": 0.1812086617740328, + "eval_logits/chosen": -2.5230772495269775, + "eval_logits/rejected": -2.9182300567626953, + "eval_logps/chosen": -0.2819940149784088, + "eval_logps/rejected": -0.48641347885131836, + "eval_loss": 0.8070082664489746, + "eval_rewards/accuracies": 0.5803571343421936, + "eval_rewards/chosen": -0.4229910671710968, + "eval_rewards/margins": 0.3066291809082031, + "eval_rewards/rejected": -0.7296201586723328, + "eval_runtime": 30.3523, + "eval_samples_per_second": 29.388, + "eval_steps_per_second": 3.69, + "step": 250 + }, + { + "epoch": 0.18845700824499412, + "grad_norm": 0.3010155260562897, + "learning_rate": 4.638410650401267e-06, + "logits/chosen": -2.4178171157836914, + "logits/rejected": -2.914424419403076, + "logps/chosen": -0.24933743476867676, + "logps/rejected": -0.4757954478263855, + "loss": 6.5743, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.3740061819553375, + "rewards/margins": 0.33968693017959595, + "rewards/rejected": -0.7136931419372559, + "step": 260 + }, + { + "epoch": 0.19570535471595543, + "grad_norm": 0.3765665888786316, + "learning_rate": 4.610819813755038e-06, + "logits/chosen": -2.407275676727295, + "logits/rejected": -2.886014938354492, + "logps/chosen": -0.27837619185447693, + "logps/rejected": -0.6118712425231934, + "loss": 6.4931, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.417564332485199, + "rewards/margins": 0.5002425909042358, + "rewards/rejected": -0.9178068041801453, + "step": 270 + }, + { + "epoch": 0.20295370118691675, + "grad_norm": 0.29865291714668274, + "learning_rate": 4.582303101775249e-06, + "logits/chosen": -2.5404791831970215, + "logits/rejected": -2.861341953277588, + "logps/chosen": -0.3124132752418518, + "logps/rejected": -0.558597207069397, + "loss": 6.2272, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.4686199128627777, + "rewards/margins": 0.3692759573459625, + "rewards/rejected": -0.8378958702087402, + "step": 280 + }, + { + "epoch": 0.21020204765787803, + "grad_norm": 0.4311545789241791, + "learning_rate": 4.55287302283426e-06, + "logits/chosen": -2.393683910369873, + "logits/rejected": -2.833466053009033, + "logps/chosen": -0.3574284017086029, + "logps/rejected": -0.8571383357048035, + "loss": 5.7937, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.5361425876617432, + "rewards/margins": 0.7495648264884949, + "rewards/rejected": -1.2857074737548828, + "step": 290 + }, + { + "epoch": 0.21745039412883935, + "grad_norm": 0.7148597836494446, + "learning_rate": 4.522542485937369e-06, + "logits/chosen": -2.369452476501465, + "logits/rejected": -2.7431299686431885, + "logps/chosen": -0.46852874755859375, + "logps/rejected": -0.9673423767089844, + "loss": 5.741, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7027931213378906, + "rewards/margins": 0.7482204437255859, + "rewards/rejected": -1.4510136842727661, + "step": 300 + }, + { + "epoch": 0.21745039412883935, + "eval_logits/chosen": -2.5259616374969482, + "eval_logits/rejected": -2.8154189586639404, + "eval_logps/chosen": -0.5109023451805115, + "eval_logps/rejected": -0.9421015381813049, + "eval_loss": 0.6990054845809937, + "eval_rewards/accuracies": 0.5714285969734192, + "eval_rewards/chosen": -0.7663536071777344, + "eval_rewards/margins": 0.6467987895011902, + "eval_rewards/rejected": -1.4131524562835693, + "eval_runtime": 30.3501, + "eval_samples_per_second": 29.39, + "eval_steps_per_second": 3.69, + "step": 300 + }, + { + "epoch": 0.22469874059980066, + "grad_norm": 0.37795087695121765, + "learning_rate": 4.491324795060491e-06, + "logits/chosen": -2.5089354515075684, + "logits/rejected": -2.827730894088745, + "logps/chosen": -0.4125432074069977, + "logps/rejected": -1.0391974449157715, + "loss": 5.3005, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6188148260116577, + "rewards/margins": 0.9399812817573547, + "rewards/rejected": -1.5587961673736572, + "step": 310 + }, + { + "epoch": 0.23194708707076198, + "grad_norm": 0.38493892550468445, + "learning_rate": 4.4592336433146e-06, + "logits/chosen": -2.3596529960632324, + "logits/rejected": -2.8088250160217285, + "logps/chosen": -0.5338706970214844, + "logps/rejected": -1.2868502140045166, + "loss": 5.2832, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.8008060455322266, + "rewards/margins": 1.1294692754745483, + "rewards/rejected": -1.930275321006775, + "step": 320 + }, + { + "epoch": 0.2391954335417233, + "grad_norm": 0.6237270832061768, + "learning_rate": 4.426283106939474e-06, + "logits/chosen": -2.5265612602233887, + "logits/rejected": -2.7830424308776855, + "logps/chosen": -0.6278705596923828, + "logps/rejected": -1.4297993183135986, + "loss": 5.1977, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.9418058395385742, + "rewards/margins": 1.2028930187225342, + "rewards/rejected": -2.1446988582611084, + "step": 330 + }, + { + "epoch": 0.2464437800126846, + "grad_norm": 0.9095363020896912, + "learning_rate": 4.3924876391293915e-06, + "logits/chosen": -2.5918514728546143, + "logits/rejected": -2.8718667030334473, + "logps/chosen": -0.6412171125411987, + "logps/rejected": -1.508195161819458, + "loss": 4.7181, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.9618256688117981, + "rewards/margins": 1.3004668951034546, + "rewards/rejected": -2.2622926235198975, + "step": 340 + }, + { + "epoch": 0.2536921264836459, + "grad_norm": 0.6144569516181946, + "learning_rate": 4.357862063693486e-06, + "logits/chosen": -2.492586612701416, + "logits/rejected": -2.9216372966766357, + "logps/chosen": -0.7246330976486206, + "logps/rejected": -2.0466060638427734, + "loss": 4.6111, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.0869497060775757, + "rewards/margins": 1.9829593896865845, + "rewards/rejected": -3.06990909576416, + "step": 350 + }, + { + "epoch": 0.2536921264836459, + "eval_logits/chosen": -2.7258641719818115, + "eval_logits/rejected": -2.9883599281311035, + "eval_logps/chosen": -0.7502545118331909, + "eval_logps/rejected": -1.7796010971069336, + "eval_loss": 0.571855366230011, + "eval_rewards/accuracies": 0.5892857313156128, + "eval_rewards/chosen": -1.1253817081451416, + "eval_rewards/margins": 1.5440199375152588, + "eval_rewards/rejected": -2.6694016456604004, + "eval_runtime": 30.3197, + "eval_samples_per_second": 29.42, + "eval_steps_per_second": 3.694, + "step": 350 + }, + { + "epoch": 0.2609404729546072, + "grad_norm": 0.6816007494926453, + "learning_rate": 4.322421568553529e-06, + "logits/chosen": -2.6618990898132324, + "logits/rejected": -2.9197638034820557, + "logps/chosen": -0.7783921360969543, + "logps/rejected": -1.9845831394195557, + "loss": 4.8991, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.1675881147384644, + "rewards/margins": 1.8092864751815796, + "rewards/rejected": -2.976874589920044, + "step": 360 + }, + { + "epoch": 0.26818881942556855, + "grad_norm": 0.5186926126480103, + "learning_rate": 4.286181699082008e-06, + "logits/chosen": -2.690049171447754, + "logits/rejected": -2.9129927158355713, + "logps/chosen": -0.7160422205924988, + "logps/rejected": -2.097799777984619, + "loss": 4.6471, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.0740633010864258, + "rewards/margins": 2.0726354122161865, + "rewards/rejected": -3.1466989517211914, + "step": 370 + }, + { + "epoch": 0.27543716589652983, + "grad_norm": 0.7122822403907776, + "learning_rate": 4.249158351283414e-06, + "logits/chosen": -2.6984095573425293, + "logits/rejected": -2.8411834239959717, + "logps/chosen": -0.724581778049469, + "logps/rejected": -2.1322455406188965, + "loss": 4.4326, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.0868725776672363, + "rewards/margins": 2.1114957332611084, + "rewards/rejected": -3.198368549346924, + "step": 380 + }, + { + "epoch": 0.2826855123674912, + "grad_norm": 0.610661506652832, + "learning_rate": 4.211367764821722e-06, + "logits/chosen": -2.5932717323303223, + "logits/rejected": -2.855365514755249, + "logps/chosen": -0.6430361866950989, + "logps/rejected": -2.8457767963409424, + "loss": 4.3874, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.9645543098449707, + "rewards/margins": 3.3041110038757324, + "rewards/rejected": -4.268665313720703, + "step": 390 + }, + { + "epoch": 0.28993385883845246, + "grad_norm": 0.627453088760376, + "learning_rate": 4.172826515897146e-06, + "logits/chosen": -2.46079683303833, + "logits/rejected": -2.839109182357788, + "logps/chosen": -0.8076715469360352, + "logps/rejected": -2.6943671703338623, + "loss": 4.2388, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.2115072011947632, + "rewards/margins": 2.8300435543060303, + "rewards/rejected": -4.041550636291504, + "step": 400 + }, + { + "epoch": 0.28993385883845246, + "eval_logits/chosen": -2.5736935138702393, + "eval_logits/rejected": -2.8549301624298096, + "eval_logps/chosen": -0.8027606010437012, + "eval_logps/rejected": -2.407195568084717, + "eval_loss": 0.49557629227638245, + "eval_rewards/accuracies": 0.6071428656578064, + "eval_rewards/chosen": -1.2041409015655518, + "eval_rewards/margins": 2.4066522121429443, + "eval_rewards/rejected": -3.6107935905456543, + "eval_runtime": 30.3144, + "eval_samples_per_second": 29.425, + "eval_steps_per_second": 3.695, + "step": 400 + }, + { + "epoch": 0.2971822053094138, + "grad_norm": 0.582290768623352, + "learning_rate": 4.133551509975264e-06, + "logits/chosen": -2.5232419967651367, + "logits/rejected": -2.8139169216156006, + "logps/chosen": -0.7921724319458008, + "logps/rejected": -2.5887393951416016, + "loss": 3.9161, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -1.1882586479187012, + "rewards/margins": 2.694850206375122, + "rewards/rejected": -3.883108615875244, + "step": 410 + }, + { + "epoch": 0.3044305517803751, + "grad_norm": 1.0708731412887573, + "learning_rate": 4.093559974371725e-06, + "logits/chosen": -2.402315616607666, + "logits/rejected": -2.6774837970733643, + "logps/chosen": -0.9347984194755554, + "logps/rejected": -3.4818978309631348, + "loss": 3.8304, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -1.4021978378295898, + "rewards/margins": 3.8206489086151123, + "rewards/rejected": -5.222846508026123, + "step": 420 + }, + { + "epoch": 0.31167889825133643, + "grad_norm": 0.6126013398170471, + "learning_rate": 4.052869450695776e-06, + "logits/chosen": -2.4616169929504395, + "logits/rejected": -2.791163921356201, + "logps/chosen": -1.073099970817566, + "logps/rejected": -3.7586822509765625, + "loss": 3.7662, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.609649896621704, + "rewards/margins": 4.028374671936035, + "rewards/rejected": -5.63802433013916, + "step": 430 + }, + { + "epoch": 0.3189272447222977, + "grad_norm": 0.6995553374290466, + "learning_rate": 4.011497787155938e-06, + "logits/chosen": -2.350310802459717, + "logits/rejected": -2.750123977661133, + "logps/chosen": -1.2352540493011475, + "logps/rejected": -4.180838584899902, + "loss": 3.4493, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.852880835533142, + "rewards/margins": 4.418376922607422, + "rewards/rejected": -6.271258354187012, + "step": 440 + }, + { + "epoch": 0.32617559119325906, + "grad_norm": 0.7922266721725464, + "learning_rate": 3.969463130731183e-06, + "logits/chosen": -2.368863344192505, + "logits/rejected": -2.7451913356781006, + "logps/chosen": -1.0907175540924072, + "logps/rejected": -4.085494041442871, + "loss": 4.0187, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.6360763311386108, + "rewards/margins": 4.4921650886535645, + "rewards/rejected": -6.128241062164307, + "step": 450 + }, + { + "epoch": 0.32617559119325906, + "eval_logits/chosen": -2.44749116897583, + "eval_logits/rejected": -2.7382781505584717, + "eval_logps/chosen": -1.3378121852874756, + "eval_logps/rejected": -3.6518471240997314, + "eval_loss": 0.45260879397392273, + "eval_rewards/accuracies": 0.6428571343421936, + "eval_rewards/chosen": -2.006718158721924, + "eval_rewards/margins": 3.4710519313812256, + "eval_rewards/rejected": -5.4777703285217285, + "eval_runtime": 30.3269, + "eval_samples_per_second": 29.413, + "eval_steps_per_second": 3.693, + "step": 450 + }, + { + "epoch": 0.33342393766422035, + "grad_norm": 0.7150818109512329, + "learning_rate": 3.92678391921108e-06, + "logits/chosen": -2.4681639671325684, + "logits/rejected": -2.797795057296753, + "logps/chosen": -1.4199602603912354, + "logps/rejected": -4.342296600341797, + "loss": 3.5788, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.1299405097961426, + "rewards/margins": 4.3835039138793945, + "rewards/rejected": -6.513444423675537, + "step": 460 + }, + { + "epoch": 0.34067228413518164, + "grad_norm": 1.260711431503296, + "learning_rate": 3.88347887310836e-06, + "logits/chosen": -2.4320380687713623, + "logits/rejected": -2.749352216720581, + "logps/chosen": -1.318566918373108, + "logps/rejected": -3.7736339569091797, + "loss": 3.5681, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.977850317955017, + "rewards/margins": 3.682600498199463, + "rewards/rejected": -5.6604509353637695, + "step": 470 + }, + { + "epoch": 0.347920630606143, + "grad_norm": 0.9010512828826904, + "learning_rate": 3.839566987447492e-06, + "logits/chosen": -2.3840298652648926, + "logits/rejected": -2.7868971824645996, + "logps/chosen": -1.6358540058135986, + "logps/rejected": -5.068122386932373, + "loss": 3.2884, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -2.4537811279296875, + "rewards/margins": 5.148402690887451, + "rewards/rejected": -7.602184295654297, + "step": 480 + }, + { + "epoch": 0.35516897707710426, + "grad_norm": 0.4977961778640747, + "learning_rate": 3.795067523432826e-06, + "logits/chosen": -2.4045886993408203, + "logits/rejected": -2.813812494277954, + "logps/chosen": -1.5761570930480957, + "logps/rejected": -5.026966094970703, + "loss": 3.5505, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.3642354011535645, + "rewards/margins": 5.176213264465332, + "rewards/rejected": -7.5404486656188965, + "step": 490 + }, + { + "epoch": 0.3624173235480656, + "grad_norm": 0.9888324737548828, + "learning_rate": 3.7500000000000005e-06, + "logits/chosen": -2.4877827167510986, + "logits/rejected": -2.7923107147216797, + "logps/chosen": -2.0706987380981445, + "logps/rejected": -5.011548042297363, + "loss": 3.3992, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -3.106048107147217, + "rewards/margins": 4.411273002624512, + "rewards/rejected": -7.517321586608887, + "step": 500 + }, + { + "epoch": 0.3624173235480656, + "eval_logits/chosen": -2.5850253105163574, + "eval_logits/rejected": -2.898167133331299, + "eval_logps/chosen": -2.3814480304718018, + "eval_logps/rejected": -5.132807731628418, + "eval_loss": 0.40332508087158203, + "eval_rewards/accuracies": 0.7142857313156128, + "eval_rewards/chosen": -3.572171926498413, + "eval_rewards/margins": 4.127039432525635, + "eval_rewards/rejected": -7.699211597442627, + "eval_runtime": 30.324, + "eval_samples_per_second": 29.416, + "eval_steps_per_second": 3.693, + "step": 500 + }, + { + "epoch": 0.3696656700190269, + "grad_norm": 0.9108109474182129, + "learning_rate": 3.7043841852542884e-06, + "logits/chosen": -2.4516501426696777, + "logits/rejected": -2.863548755645752, + "logps/chosen": -2.235166549682617, + "logps/rejected": -5.824828147888184, + "loss": 3.2045, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -3.352750301361084, + "rewards/margins": 5.384493350982666, + "rewards/rejected": -8.737241744995117, + "step": 510 + }, + { + "epoch": 0.37691401648998824, + "grad_norm": 1.608782410621643, + "learning_rate": 3.658240087799655e-06, + "logits/chosen": -2.448782444000244, + "logits/rejected": -2.9055087566375732, + "logps/chosen": -2.345585584640503, + "logps/rejected": -5.665570259094238, + "loss": 3.2997, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -3.518378496170044, + "rewards/margins": 4.979977130889893, + "rewards/rejected": -8.498355865478516, + "step": 520 + }, + { + "epoch": 0.3841623629609495, + "grad_norm": 2.4802284240722656, + "learning_rate": 3.611587947962319e-06, + "logits/chosen": -2.5400586128234863, + "logits/rejected": -2.9653382301330566, + "logps/chosen": -2.9243946075439453, + "logps/rejected": -7.067171573638916, + "loss": 2.7084, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -4.386591911315918, + "rewards/margins": 6.214165210723877, + "rewards/rejected": -10.60075569152832, + "step": 530 + }, + { + "epoch": 0.39141070943191086, + "grad_norm": 1.4558824300765991, + "learning_rate": 3.564448228912682e-06, + "logits/chosen": -2.538511276245117, + "logits/rejected": -3.0383963584899902, + "logps/chosen": -3.83245849609375, + "logps/rejected": -7.150879859924316, + "loss": 2.85, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -5.748687744140625, + "rewards/margins": 4.977631568908691, + "rewards/rejected": -10.726319313049316, + "step": 540 + }, + { + "epoch": 0.39865905590287215, + "grad_norm": 6.12661600112915, + "learning_rate": 3.516841607689501e-06, + "logits/chosen": -2.5623717308044434, + "logits/rejected": -2.930835485458374, + "logps/chosen": -3.9018759727478027, + "logps/rejected": -7.186791896820068, + "loss": 2.6406, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.852814674377441, + "rewards/margins": 4.92737340927124, + "rewards/rejected": -10.78018856048584, + "step": 550 + }, + { + "epoch": 0.39865905590287215, + "eval_logits/chosen": -2.5746138095855713, + "eval_logits/rejected": -2.9623827934265137, + "eval_logps/chosen": -4.4805707931518555, + "eval_logps/rejected": -7.7552666664123535, + "eval_loss": 0.30369508266448975, + "eval_rewards/accuracies": 0.8660714030265808, + "eval_rewards/chosen": -6.720856189727783, + "eval_rewards/margins": 4.912044048309326, + "eval_rewards/rejected": -11.632901191711426, + "eval_runtime": 30.3366, + "eval_samples_per_second": 29.403, + "eval_steps_per_second": 3.692, + "step": 550 + }, + { + "epoch": 0.4059074023738335, + "grad_norm": 2.2883663177490234, + "learning_rate": 3.4687889661302577e-06, + "logits/chosen": -2.523510456085205, + "logits/rejected": -3.0058722496032715, + "logps/chosen": -4.60703706741333, + "logps/rejected": -8.818490982055664, + "loss": 2.1664, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.9105544090271, + "rewards/margins": 6.3171820640563965, + "rewards/rejected": -13.22773551940918, + "step": 560 + }, + { + "epoch": 0.4131557488447948, + "grad_norm": 3.3689115047454834, + "learning_rate": 3.4203113817116955e-06, + "logits/chosen": -2.592515230178833, + "logits/rejected": -2.9502272605895996, + "logps/chosen": -5.648528099060059, + "logps/rejected": -10.127589225769043, + "loss": 2.2641, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -8.472792625427246, + "rewards/margins": 6.718589782714844, + "rewards/rejected": -15.191381454467773, + "step": 570 + }, + { + "epoch": 0.42040409531575607, + "grad_norm": 4.080602169036865, + "learning_rate": 3.3714301183045382e-06, + "logits/chosen": -2.418156862258911, + "logits/rejected": -3.0541069507598877, + "logps/chosen": -5.756854057312012, + "logps/rejected": -11.639761924743652, + "loss": 2.3099, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -8.63528060913086, + "rewards/margins": 8.82436466217041, + "rewards/rejected": -17.459644317626953, + "step": 580 + }, + { + "epoch": 0.4276524417867174, + "grad_norm": 1.4410984516143799, + "learning_rate": 3.3221666168464584e-06, + "logits/chosen": -2.506753921508789, + "logits/rejected": -3.029228687286377, + "logps/chosen": -6.3516435623168945, + "logps/rejected": -11.839346885681152, + "loss": 2.0248, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -9.527464866638184, + "rewards/margins": 8.231555938720703, + "rewards/rejected": -17.75901985168457, + "step": 590 + }, + { + "epoch": 0.4349007882576787, + "grad_norm": 3.3688838481903076, + "learning_rate": 3.272542485937369e-06, + "logits/chosen": -2.4242749214172363, + "logits/rejected": -2.9992825984954834, + "logps/chosen": -6.2571916580200195, + "logps/rejected": -12.356736183166504, + "loss": 2.0191, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -9.385787010192871, + "rewards/margins": 9.149316787719727, + "rewards/rejected": -18.535104751586914, + "step": 600 + }, + { + "epoch": 0.4349007882576787, + "eval_logits/chosen": -2.5597894191741943, + "eval_logits/rejected": -2.9679033756256104, + "eval_logps/chosen": -6.51260232925415, + "eval_logps/rejected": -11.510099411010742, + "eval_loss": 0.22690872848033905, + "eval_rewards/accuracies": 0.9285714030265808, + "eval_rewards/chosen": -9.768902778625488, + "eval_rewards/margins": 7.496245384216309, + "eval_rewards/rejected": -17.26515007019043, + "eval_runtime": 30.2947, + "eval_samples_per_second": 29.444, + "eval_steps_per_second": 3.697, + "step": 600 + }, + { + "epoch": 0.44214913472864004, + "grad_norm": 7.5001959800720215, + "learning_rate": 3.222579492361179e-06, + "logits/chosen": -2.594581127166748, + "logits/rejected": -3.0206668376922607, + "logps/chosen": -7.0802435874938965, + "logps/rejected": -12.667734146118164, + "loss": 1.6936, + "rewards/accuracies": 0.9375, + "rewards/chosen": -10.620367050170898, + "rewards/margins": 8.381233215332031, + "rewards/rejected": -19.001598358154297, + "step": 610 + }, + { + "epoch": 0.4493974811996013, + "grad_norm": 2.639087677001953, + "learning_rate": 3.1722995515381644e-06, + "logits/chosen": -2.511791229248047, + "logits/rejected": -3.0638930797576904, + "logps/chosen": -7.493863105773926, + "logps/rejected": -14.954449653625488, + "loss": 1.6215, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -11.24079418182373, + "rewards/margins": 11.19088363647461, + "rewards/rejected": -22.431676864624023, + "step": 620 + }, + { + "epoch": 0.45664582767056267, + "grad_norm": 5.2657270431518555, + "learning_rate": 3.121724717912138e-06, + "logits/chosen": -2.5814051628112793, + "logits/rejected": -2.963531017303467, + "logps/chosen": -8.267634391784668, + "logps/rejected": -14.717096328735352, + "loss": 1.7936, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -12.40145206451416, + "rewards/margins": 9.6741943359375, + "rewards/rejected": -22.075647354125977, + "step": 630 + }, + { + "epoch": 0.46389417414152395, + "grad_norm": 8.628580093383789, + "learning_rate": 3.0708771752766397e-06, + "logits/chosen": -2.436100482940674, + "logits/rejected": -2.9183783531188965, + "logps/chosen": -7.733956336975098, + "logps/rejected": -15.915423393249512, + "loss": 1.7631, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -11.600934028625488, + "rewards/margins": 12.272205352783203, + "rewards/rejected": -23.873138427734375, + "step": 640 + }, + { + "epoch": 0.4711425206124853, + "grad_norm": 3.072305679321289, + "learning_rate": 3.019779227044398e-06, + "logits/chosen": -2.4763903617858887, + "logits/rejected": -2.9726059436798096, + "logps/chosen": -6.810072422027588, + "logps/rejected": -13.924298286437988, + "loss": 1.5531, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -10.215107917785645, + "rewards/margins": 10.671339988708496, + "rewards/rejected": -20.88644790649414, + "step": 650 + }, + { + "epoch": 0.4711425206124853, + "eval_logits/chosen": -2.459520101547241, + "eval_logits/rejected": -2.852581024169922, + "eval_logps/chosen": -6.569087028503418, + "eval_logps/rejected": -12.969863891601562, + "eval_loss": 0.21141496300697327, + "eval_rewards/accuracies": 0.9285714030265808, + "eval_rewards/chosen": -9.853631019592285, + "eval_rewards/margins": 9.601165771484375, + "eval_rewards/rejected": -19.454797744750977, + "eval_runtime": 30.32, + "eval_samples_per_second": 29.42, + "eval_steps_per_second": 3.694, + "step": 650 + }, + { + "epoch": 0.4783908670834466, + "grad_norm": 3.4517831802368164, + "learning_rate": 2.9684532864643123e-06, + "logits/chosen": -2.5354456901550293, + "logits/rejected": -2.840172290802002, + "logps/chosen": -7.238469123840332, + "logps/rejected": -14.787511825561523, + "loss": 1.3669, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -10.857705116271973, + "rewards/margins": 11.323565483093262, + "rewards/rejected": -22.181270599365234, + "step": 660 + }, + { + "epoch": 0.4856392135544079, + "grad_norm": 8.935027122497559, + "learning_rate": 2.9169218667902562e-06, + "logits/chosen": -2.47978138923645, + "logits/rejected": -2.9299492835998535, + "logps/chosen": -7.6814703941345215, + "logps/rejected": -15.477340698242188, + "loss": 1.8781, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -11.522205352783203, + "rewards/margins": 11.693807601928711, + "rewards/rejected": -23.216012954711914, + "step": 670 + }, + { + "epoch": 0.4928875600253692, + "grad_norm": 1.7096459865570068, + "learning_rate": 2.8652075714060296e-06, + "logits/chosen": -2.4414560794830322, + "logits/rejected": -2.864112377166748, + "logps/chosen": -8.507112503051758, + "logps/rejected": -15.909780502319336, + "loss": 1.2831, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -12.76066780090332, + "rewards/margins": 11.10400104522705, + "rewards/rejected": -23.864669799804688, + "step": 680 + }, + { + "epoch": 0.5001359064963306, + "grad_norm": 3.814954996109009, + "learning_rate": 2.813333083910761e-06, + "logits/chosen": -2.4312682151794434, + "logits/rejected": -2.993825674057007, + "logps/chosen": -8.516946792602539, + "logps/rejected": -17.835708618164062, + "loss": 1.0306, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -12.775420188903809, + "rewards/margins": 13.978144645690918, + "rewards/rejected": -26.75356674194336, + "step": 690 + }, + { + "epoch": 0.5073842529672918, + "grad_norm": 6.554664611816406, + "learning_rate": 2.761321158169134e-06, + "logits/chosen": -2.458709239959717, + "logits/rejected": -2.9223692417144775, + "logps/chosen": -9.111552238464355, + "logps/rejected": -19.193201065063477, + "loss": 1.1941, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -13.667327880859375, + "rewards/margins": 15.122472763061523, + "rewards/rejected": -28.7898006439209, + "step": 700 + }, + { + "epoch": 0.5073842529672918, + "eval_logits/chosen": -2.5367431640625, + "eval_logits/rejected": -2.9102938175201416, + "eval_logps/chosen": -10.25200366973877, + "eval_logps/rejected": -17.488019943237305, + "eval_loss": 0.1846819519996643, + "eval_rewards/accuracies": 0.9285714030265808, + "eval_rewards/chosen": -15.378005027770996, + "eval_rewards/margins": 10.854025840759277, + "eval_rewards/rejected": -26.232030868530273, + "eval_runtime": 30.3487, + "eval_samples_per_second": 29.392, + "eval_steps_per_second": 3.69, + "step": 700 + }, + { + "epoch": 0.5146325994382531, + "grad_norm": 11.16737174987793, + "learning_rate": 2.70919460833079e-06, + "logits/chosen": -2.520266056060791, + "logits/rejected": -2.9030609130859375, + "logps/chosen": -10.487688064575195, + "logps/rejected": -18.03569984436035, + "loss": 1.7604, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -15.731532096862793, + "rewards/margins": 11.322016716003418, + "rewards/rejected": -27.053552627563477, + "step": 710 + }, + { + "epoch": 0.5218809459092144, + "grad_norm": 8.75973129272461, + "learning_rate": 2.6569762988232838e-06, + "logits/chosen": -2.427156686782837, + "logits/rejected": -2.9073309898376465, + "logps/chosen": -8.983491897583008, + "logps/rejected": -18.08041000366211, + "loss": 1.2509, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -13.475237846374512, + "rewards/margins": 13.64537525177002, + "rewards/rejected": -27.1206111907959, + "step": 720 + }, + { + "epoch": 0.5291292923801758, + "grad_norm": 3.2157235145568848, + "learning_rate": 2.604689134322999e-06, + "logits/chosen": -2.4182469844818115, + "logits/rejected": -2.941476821899414, + "logps/chosen": -8.248749732971191, + "logps/rejected": -16.914398193359375, + "loss": 1.2837, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -12.373125076293945, + "rewards/margins": 12.99847412109375, + "rewards/rejected": -25.371599197387695, + "step": 730 + }, + { + "epoch": 0.5363776388511371, + "grad_norm": 3.8172707557678223, + "learning_rate": 2.5523560497083927e-06, + "logits/chosen": -2.4228386878967285, + "logits/rejected": -2.9412994384765625, + "logps/chosen": -7.712450981140137, + "logps/rejected": -16.946636199951172, + "loss": 1.2981, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -11.56867790222168, + "rewards/margins": 13.851278305053711, + "rewards/rejected": -25.41995620727539, + "step": 740 + }, + { + "epoch": 0.5436259853220984, + "grad_norm": 3.1835663318634033, + "learning_rate": 2.5e-06, + "logits/chosen": -2.413264036178589, + "logits/rejected": -2.8962299823760986, + "logps/chosen": -7.1312408447265625, + "logps/rejected": -17.016965866088867, + "loss": 1.1198, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -10.69686222076416, + "rewards/margins": 14.828587532043457, + "rewards/rejected": -25.52545166015625, + "step": 750 + }, + { + "epoch": 0.5436259853220984, + "eval_logits/chosen": -2.482361316680908, + "eval_logits/rejected": -2.858593702316284, + "eval_logps/chosen": -8.353605270385742, + "eval_logps/rejected": -16.015188217163086, + "eval_loss": 0.15141016244888306, + "eval_rewards/accuracies": 0.9285714030265808, + "eval_rewards/chosen": -12.530407905578613, + "eval_rewards/margins": 11.4923734664917, + "eval_rewards/rejected": -24.022781372070312, + "eval_runtime": 30.3199, + "eval_samples_per_second": 29.42, + "eval_steps_per_second": 3.694, + "step": 750 + }, + { + "epoch": 0.5508743317930597, + "grad_norm": 3.896941661834717, + "learning_rate": 2.447643950291608e-06, + "logits/chosen": -2.4703681468963623, + "logits/rejected": -2.8748607635498047, + "logps/chosen": -8.088781356811523, + "logps/rejected": -16.520170211791992, + "loss": 1.1826, + "rewards/accuracies": 0.9375, + "rewards/chosen": -12.133172035217285, + "rewards/margins": 12.647082328796387, + "rewards/rejected": -24.780254364013672, + "step": 760 + }, + { + "epoch": 0.5581226782640211, + "grad_norm": 7.569993495941162, + "learning_rate": 2.3953108656770018e-06, + "logits/chosen": -2.452991485595703, + "logits/rejected": -2.8747894763946533, + "logps/chosen": -9.35241413116455, + "logps/rejected": -18.470417022705078, + "loss": 1.2527, + "rewards/accuracies": 1.0, + "rewards/chosen": -14.028620719909668, + "rewards/margins": 13.67700481414795, + "rewards/rejected": -27.70562744140625, + "step": 770 + }, + { + "epoch": 0.5653710247349824, + "grad_norm": 3.0607998371124268, + "learning_rate": 2.3430237011767166e-06, + "logits/chosen": -2.4571075439453125, + "logits/rejected": -2.8483026027679443, + "logps/chosen": -7.992903232574463, + "logps/rejected": -17.579792022705078, + "loss": 0.7915, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -11.989354133605957, + "rewards/margins": 14.380337715148926, + "rewards/rejected": -26.36968994140625, + "step": 780 + }, + { + "epoch": 0.5726193712059436, + "grad_norm": 6.031089782714844, + "learning_rate": 2.290805391669212e-06, + "logits/chosen": -2.3782265186309814, + "logits/rejected": -2.7803268432617188, + "logps/chosen": -8.120704650878906, + "logps/rejected": -17.844463348388672, + "loss": 0.9819, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -12.18105697631836, + "rewards/margins": 14.585641860961914, + "rewards/rejected": -26.766698837280273, + "step": 790 + }, + { + "epoch": 0.5798677176769049, + "grad_norm": 4.0991363525390625, + "learning_rate": 2.238678841830867e-06, + "logits/chosen": -2.3170902729034424, + "logits/rejected": -2.82861328125, + "logps/chosen": -7.7450361251831055, + "logps/rejected": -17.050933837890625, + "loss": 1.0712, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -11.6175537109375, + "rewards/margins": 13.958849906921387, + "rewards/rejected": -25.576404571533203, + "step": 800 + }, + { + "epoch": 0.5798677176769049, + "eval_logits/chosen": -2.3983051776885986, + "eval_logits/rejected": -2.7665843963623047, + "eval_logps/chosen": -8.15335464477539, + "eval_logps/rejected": -16.25174331665039, + "eval_loss": 0.12996967136859894, + "eval_rewards/accuracies": 0.9464285969734192, + "eval_rewards/chosen": -12.230032920837402, + "eval_rewards/margins": 12.14758586883545, + "eval_rewards/rejected": -24.377620697021484, + "eval_runtime": 30.3735, + "eval_samples_per_second": 29.368, + "eval_steps_per_second": 3.687, + "step": 800 + }, + { + "epoch": 0.5871160641478663, + "grad_norm": 5.751565456390381, + "learning_rate": 2.186666916089239e-06, + "logits/chosen": -2.3348138332366943, + "logits/rejected": -2.769927501678467, + "logps/chosen": -8.014350891113281, + "logps/rejected": -16.63930892944336, + "loss": 1.2619, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -12.021525382995605, + "rewards/margins": 12.93743896484375, + "rewards/rejected": -24.958965301513672, + "step": 810 + }, + { + "epoch": 0.5943644106188276, + "grad_norm": 7.957540512084961, + "learning_rate": 2.134792428593971e-06, + "logits/chosen": -2.327152967453003, + "logits/rejected": -2.796149253845215, + "logps/chosen": -7.6543426513671875, + "logps/rejected": -16.770008087158203, + "loss": 1.0105, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -11.481512069702148, + "rewards/margins": 13.673500061035156, + "rewards/rejected": -25.155010223388672, + "step": 820 + }, + { + "epoch": 0.6016127570897889, + "grad_norm": 3.2892773151397705, + "learning_rate": 2.0830781332097446e-06, + "logits/chosen": -2.3687503337860107, + "logits/rejected": -2.8093888759613037, + "logps/chosen": -9.138994216918945, + "logps/rejected": -18.48851776123047, + "loss": 0.8295, + "rewards/accuracies": 0.9375, + "rewards/chosen": -13.708491325378418, + "rewards/margins": 14.024286270141602, + "rewards/rejected": -27.732776641845703, + "step": 830 + }, + { + "epoch": 0.6088611035607502, + "grad_norm": 3.2489373683929443, + "learning_rate": 2.031546713535688e-06, + "logits/chosen": -2.454373836517334, + "logits/rejected": -2.787046432495117, + "logps/chosen": -8.555706977844238, + "logps/rejected": -17.804542541503906, + "loss": 0.9398, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -12.833561897277832, + "rewards/margins": 13.873250961303711, + "rewards/rejected": -26.70681381225586, + "step": 840 + }, + { + "epoch": 0.6161094500317115, + "grad_norm": 4.858041286468506, + "learning_rate": 1.9802207729556023e-06, + "logits/chosen": -2.3334898948669434, + "logits/rejected": -2.799832344055176, + "logps/chosen": -8.557367324829102, + "logps/rejected": -18.578689575195312, + "loss": 0.8919, + "rewards/accuracies": 1.0, + "rewards/chosen": -12.836050033569336, + "rewards/margins": 15.031984329223633, + "rewards/rejected": -27.868032455444336, + "step": 850 + }, + { + "epoch": 0.6161094500317115, + "eval_logits/chosen": -2.394418716430664, + "eval_logits/rejected": -2.768528461456299, + "eval_logps/chosen": -8.943707466125488, + "eval_logps/rejected": -17.889659881591797, + "eval_loss": 0.11857374012470245, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -13.415560722351074, + "eval_rewards/margins": 13.418928146362305, + "eval_rewards/rejected": -26.834491729736328, + "eval_runtime": 30.3214, + "eval_samples_per_second": 29.418, + "eval_steps_per_second": 3.694, + "step": 850 + }, + { + "epoch": 0.6233577965026729, + "grad_norm": 6.336078643798828, + "learning_rate": 1.9291228247233607e-06, + "logits/chosen": -2.3998782634735107, + "logits/rejected": -2.8122310638427734, + "logps/chosen": -8.910730361938477, + "logps/rejected": -18.417646408081055, + "loss": 0.9668, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -13.366094589233398, + "rewards/margins": 14.26037883758545, + "rewards/rejected": -27.626474380493164, + "step": 860 + }, + { + "epoch": 0.6306061429736342, + "grad_norm": 12.80254077911377, + "learning_rate": 1.8782752820878636e-06, + "logits/chosen": -2.3551506996154785, + "logits/rejected": -2.8078038692474365, + "logps/chosen": -7.851692199707031, + "logps/rejected": -18.801013946533203, + "loss": 0.623, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -11.77753734588623, + "rewards/margins": 16.42398452758789, + "rewards/rejected": -28.201519012451172, + "step": 870 + }, + { + "epoch": 0.6378544894445954, + "grad_norm": 3.489668369293213, + "learning_rate": 1.827700448461836e-06, + "logits/chosen": -2.4149603843688965, + "logits/rejected": -2.790621519088745, + "logps/chosen": -8.617798805236816, + "logps/rejected": -19.069250106811523, + "loss": 0.8222, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -12.926698684692383, + "rewards/margins": 15.677175521850586, + "rewards/rejected": -28.6038761138916, + "step": 880 + }, + { + "epoch": 0.6451028359155567, + "grad_norm": 1.9104561805725098, + "learning_rate": 1.7774205076388207e-06, + "logits/chosen": -2.3620147705078125, + "logits/rejected": -2.8059632778167725, + "logps/chosen": -9.715843200683594, + "logps/rejected": -21.32562828063965, + "loss": 0.7366, + "rewards/accuracies": 1.0, + "rewards/chosen": -14.573763847351074, + "rewards/margins": 17.414676666259766, + "rewards/rejected": -31.98844337463379, + "step": 890 + }, + { + "epoch": 0.6523511823865181, + "grad_norm": 9.600937843322754, + "learning_rate": 1.7274575140626318e-06, + "logits/chosen": -2.49873948097229, + "logits/rejected": -2.825862169265747, + "logps/chosen": -10.357137680053711, + "logps/rejected": -19.829532623291016, + "loss": 1.1936, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -15.53570556640625, + "rewards/margins": 14.208595275878906, + "rewards/rejected": -29.744298934936523, + "step": 900 + }, + { + "epoch": 0.6523511823865181, + "eval_logits/chosen": -2.4111170768737793, + "eval_logits/rejected": -2.7982914447784424, + "eval_logps/chosen": -10.249353408813477, + "eval_logps/rejected": -20.502059936523438, + "eval_loss": 0.11673425137996674, + "eval_rewards/accuracies": 0.9464285969734192, + "eval_rewards/chosen": -15.374032020568848, + "eval_rewards/margins": 15.379058837890625, + "eval_rewards/rejected": -30.75309181213379, + "eval_runtime": 30.3049, + "eval_samples_per_second": 29.434, + "eval_steps_per_second": 3.696, + "step": 900 + }, + { + "epoch": 0.6595995288574794, + "grad_norm": 4.28416633605957, + "learning_rate": 1.677833383153542e-06, + "logits/chosen": -2.348818063735962, + "logits/rejected": -2.8150997161865234, + "logps/chosen": -9.674580574035645, + "logps/rejected": -21.100772857666016, + "loss": 0.6817, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -14.511869430541992, + "rewards/margins": 17.1392879486084, + "rewards/rejected": -31.65115737915039, + "step": 910 + }, + { + "epoch": 0.6668478753284407, + "grad_norm": 6.507571220397949, + "learning_rate": 1.6285698816954626e-06, + "logits/chosen": -2.3171777725219727, + "logits/rejected": -2.7714550495147705, + "logps/chosen": -8.589642524719238, + "logps/rejected": -20.672466278076172, + "loss": 0.8751, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -12.8844633102417, + "rewards/margins": 18.124235153198242, + "rewards/rejected": -31.008697509765625, + "step": 920 + }, + { + "epoch": 0.674096221799402, + "grad_norm": 5.07767915725708, + "learning_rate": 1.5796886182883053e-06, + "logits/chosen": -2.374326705932617, + "logits/rejected": -2.7454559803009033, + "logps/chosen": -9.055803298950195, + "logps/rejected": -21.033706665039062, + "loss": 0.8639, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -13.583703994750977, + "rewards/margins": 17.966854095458984, + "rewards/rejected": -31.550561904907227, + "step": 930 + }, + { + "epoch": 0.6813445682703633, + "grad_norm": 4.192989349365234, + "learning_rate": 1.5312110338697427e-06, + "logits/chosen": -2.4058878421783447, + "logits/rejected": -2.81569766998291, + "logps/chosen": -10.162097930908203, + "logps/rejected": -21.776790618896484, + "loss": 0.8294, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -15.243146896362305, + "rewards/margins": 17.422039031982422, + "rewards/rejected": -32.665184020996094, + "step": 940 + }, + { + "epoch": 0.6885929147413247, + "grad_norm": 6.396577835083008, + "learning_rate": 1.4831583923105e-06, + "logits/chosen": -2.4091625213623047, + "logits/rejected": -2.7940189838409424, + "logps/chosen": -9.696913719177246, + "logps/rejected": -20.56863784790039, + "loss": 0.9113, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -14.545370101928711, + "rewards/margins": 16.30758285522461, + "rewards/rejected": -30.852951049804688, + "step": 950 + }, + { + "epoch": 0.6885929147413247, + "eval_logits/chosen": -2.371931314468384, + "eval_logits/rejected": -2.7571141719818115, + "eval_logps/chosen": -8.616922378540039, + "eval_logps/rejected": -18.889474868774414, + "eval_loss": 0.1089157834649086, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -12.925383567810059, + "eval_rewards/margins": 15.40882682800293, + "eval_rewards/rejected": -28.334211349487305, + "eval_runtime": 30.3205, + "eval_samples_per_second": 29.419, + "eval_steps_per_second": 3.694, + "step": 950 + }, + { + "epoch": 0.695841261212286, + "grad_norm": 2.0531630516052246, + "learning_rate": 1.4355517710873184e-06, + "logits/chosen": -2.3074121475219727, + "logits/rejected": -2.796274423599243, + "logps/chosen": -8.068899154663086, + "logps/rejected": -18.71575164794922, + "loss": 0.8595, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -12.103349685668945, + "rewards/margins": 15.9702787399292, + "rewards/rejected": -28.07362937927246, + "step": 960 + }, + { + "epoch": 0.7030896076832472, + "grad_norm": 8.175732612609863, + "learning_rate": 1.388412052037682e-06, + "logits/chosen": -2.3601155281066895, + "logits/rejected": -2.8169026374816895, + "logps/chosen": -9.363846778869629, + "logps/rejected": -21.162973403930664, + "loss": 0.9415, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -14.045768737792969, + "rewards/margins": 17.69869041442871, + "rewards/rejected": -31.744457244873047, + "step": 970 + }, + { + "epoch": 0.7103379541542085, + "grad_norm": 21.960830688476562, + "learning_rate": 1.3417599122003464e-06, + "logits/chosen": -2.2582755088806152, + "logits/rejected": -2.8120741844177246, + "logps/chosen": -8.260191917419434, + "logps/rejected": -20.000268936157227, + "loss": 0.8961, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -12.390287399291992, + "rewards/margins": 17.610116958618164, + "rewards/rejected": -30.000402450561523, + "step": 980 + }, + { + "epoch": 0.7175863006251699, + "grad_norm": 4.283840179443359, + "learning_rate": 1.2956158147457116e-06, + "logits/chosen": -2.3035526275634766, + "logits/rejected": -2.7238144874572754, + "logps/chosen": -7.883841037750244, + "logps/rejected": -19.16309928894043, + "loss": 0.7882, + "rewards/accuracies": 1.0, + "rewards/chosen": -11.825760841369629, + "rewards/margins": 16.91888427734375, + "rewards/rejected": -28.744644165039062, + "step": 990 + }, + { + "epoch": 0.7248346470961312, + "grad_norm": 17.20541763305664, + "learning_rate": 1.2500000000000007e-06, + "logits/chosen": -2.3649654388427734, + "logits/rejected": -2.760070323944092, + "logps/chosen": -9.843507766723633, + "logps/rejected": -20.48518943786621, + "loss": 1.135, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -14.765260696411133, + "rewards/margins": 15.96252155303955, + "rewards/rejected": -30.727783203125, + "step": 1000 + }, + { + "epoch": 0.7248346470961312, + "eval_logits/chosen": -2.390676736831665, + "eval_logits/rejected": -2.7747442722320557, + "eval_logps/chosen": -9.749165534973145, + "eval_logps/rejected": -20.03530502319336, + "eval_loss": 0.1021483913064003, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -14.623747825622559, + "eval_rewards/margins": 15.42920970916748, + "eval_rewards/rejected": -30.05295753479004, + "eval_runtime": 30.3161, + "eval_samples_per_second": 29.423, + "eval_steps_per_second": 3.694, + "step": 1000 + }, + { + "epoch": 0.7320829935670925, + "grad_norm": 5.572669506072998, + "learning_rate": 1.204932476567175e-06, + "logits/chosen": -2.300497055053711, + "logits/rejected": -2.7727770805358887, + "logps/chosen": -9.08503532409668, + "logps/rejected": -21.310386657714844, + "loss": 0.769, + "rewards/accuracies": 1.0, + "rewards/chosen": -13.62755298614502, + "rewards/margins": 18.338027954101562, + "rewards/rejected": -31.965579986572266, + "step": 1010 + }, + { + "epoch": 0.7393313400380538, + "grad_norm": 1.9562855958938599, + "learning_rate": 1.160433012552508e-06, + "logits/chosen": -2.3115997314453125, + "logits/rejected": -2.8770854473114014, + "logps/chosen": -8.05511474609375, + "logps/rejected": -21.068984985351562, + "loss": 0.5362, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -12.082670211791992, + "rewards/margins": 19.52080726623535, + "rewards/rejected": -31.60347557067871, + "step": 1020 + }, + { + "epoch": 0.7465796865090152, + "grad_norm": 6.376579761505127, + "learning_rate": 1.11652112689164e-06, + "logits/chosen": -2.349559783935547, + "logits/rejected": -2.8144681453704834, + "logps/chosen": -8.122370719909668, + "logps/rejected": -19.65790557861328, + "loss": 0.8026, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -12.183557510375977, + "rewards/margins": 17.303298950195312, + "rewards/rejected": -29.48685646057129, + "step": 1030 + }, + { + "epoch": 0.7538280329799765, + "grad_norm": 9.438390731811523, + "learning_rate": 1.073216080788921e-06, + "logits/chosen": -2.3181960582733154, + "logits/rejected": -2.7197091579437256, + "logps/chosen": -7.880677700042725, + "logps/rejected": -19.085203170776367, + "loss": 0.7519, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -11.821017265319824, + "rewards/margins": 16.806785583496094, + "rewards/rejected": -28.6278018951416, + "step": 1040 + }, + { + "epoch": 0.7610763794509378, + "grad_norm": 5.646973609924316, + "learning_rate": 1.0305368692688175e-06, + "logits/chosen": -2.271108865737915, + "logits/rejected": -2.7761292457580566, + "logps/chosen": -7.734799861907959, + "logps/rejected": -19.717693328857422, + "loss": 0.5528, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -11.602200508117676, + "rewards/margins": 17.97433853149414, + "rewards/rejected": -29.5765380859375, + "step": 1050 + }, + { + "epoch": 0.7610763794509378, + "eval_logits/chosen": -2.3761844635009766, + "eval_logits/rejected": -2.758427381515503, + "eval_logps/chosen": -9.39671802520752, + "eval_logps/rejected": -19.70557975769043, + "eval_loss": 0.1007753536105156, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -14.095077514648438, + "eval_rewards/margins": 15.463290214538574, + "eval_rewards/rejected": -29.55837059020996, + "eval_runtime": 30.3239, + "eval_samples_per_second": 29.416, + "eval_steps_per_second": 3.693, + "step": 1050 + }, + { + "epoch": 0.768324725921899, + "grad_norm": 3.293710947036743, + "learning_rate": 9.88502212844063e-07, + "logits/chosen": -2.3491718769073486, + "logits/rejected": -2.7819125652313232, + "logps/chosen": -8.037856101989746, + "logps/rejected": -19.4016170501709, + "loss": 0.792, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -12.056783676147461, + "rewards/margins": 17.04564094543457, + "rewards/rejected": -29.1024227142334, + "step": 1060 + }, + { + "epoch": 0.7755730723928603, + "grad_norm": 3.9602127075195312, + "learning_rate": 9.471305493042243e-07, + "logits/chosen": -2.236292600631714, + "logits/rejected": -2.6696109771728516, + "logps/chosen": -7.1002068519592285, + "logps/rejected": -19.54210090637207, + "loss": 0.5324, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -10.650310516357422, + "rewards/margins": 18.662839889526367, + "rewards/rejected": -29.31315040588379, + "step": 1070 + }, + { + "epoch": 0.7828214188638217, + "grad_norm": 2.300848960876465, + "learning_rate": 9.064400256282757e-07, + "logits/chosen": -2.3093066215515137, + "logits/rejected": -2.6977267265319824, + "logps/chosen": -8.959060668945312, + "logps/rejected": -18.99527359008789, + "loss": 0.5667, + "rewards/accuracies": 1.0, + "rewards/chosen": -13.438589096069336, + "rewards/margins": 15.05431842803955, + "rewards/rejected": -28.492910385131836, + "step": 1080 + }, + { + "epoch": 0.790069765334783, + "grad_norm": 1.5451587438583374, + "learning_rate": 8.664484900247363e-07, + "logits/chosen": -2.414778232574463, + "logits/rejected": -2.750126838684082, + "logps/chosen": -9.486712455749512, + "logps/rejected": -20.353275299072266, + "loss": 0.6379, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -14.230069160461426, + "rewards/margins": 16.299842834472656, + "rewards/rejected": -30.529911041259766, + "step": 1090 + }, + { + "epoch": 0.7973181118057443, + "grad_norm": 7.686610221862793, + "learning_rate": 8.271734841028553e-07, + "logits/chosen": -2.383478879928589, + "logits/rejected": -2.6949422359466553, + "logps/chosen": -9.218855857849121, + "logps/rejected": -20.50551414489746, + "loss": 0.6618, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -13.828283309936523, + "rewards/margins": 16.929988861083984, + "rewards/rejected": -30.758275985717773, + "step": 1100 + }, + { + "epoch": 0.7973181118057443, + "eval_logits/chosen": -2.357304811477661, + "eval_logits/rejected": -2.741511821746826, + "eval_logps/chosen": -8.644713401794434, + "eval_logps/rejected": -19.651016235351562, + "eval_loss": 0.0923290029168129, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -12.967069625854492, + "eval_rewards/margins": 16.50945472717285, + "eval_rewards/rejected": -29.476526260375977, + "eval_runtime": 30.3348, + "eval_samples_per_second": 29.405, + "eval_steps_per_second": 3.692, + "step": 1100 + }, + { + "epoch": 0.8045664582767056, + "grad_norm": 2.6959259510040283, + "learning_rate": 7.886322351782782e-07, + "logits/chosen": -2.300200939178467, + "logits/rejected": -2.759124994277954, + "logps/chosen": -8.86776351928711, + "logps/rejected": -21.157636642456055, + "loss": 0.5303, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -13.301645278930664, + "rewards/margins": 18.434810638427734, + "rewards/rejected": -31.736459732055664, + "step": 1110 + }, + { + "epoch": 0.811814804747667, + "grad_norm": 1.7385252714157104, + "learning_rate": 7.508416487165862e-07, + "logits/chosen": -2.2611403465270996, + "logits/rejected": -2.7048158645629883, + "logps/chosen": -8.681452751159668, + "logps/rejected": -20.63593101501465, + "loss": 0.9835, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -13.022178649902344, + "rewards/margins": 17.931718826293945, + "rewards/rejected": -30.95389747619629, + "step": 1120 + }, + { + "epoch": 0.8190631512186283, + "grad_norm": 31.765613555908203, + "learning_rate": 7.138183009179922e-07, + "logits/chosen": -2.347527027130127, + "logits/rejected": -2.8267199993133545, + "logps/chosen": -7.8143181800842285, + "logps/rejected": -21.345165252685547, + "loss": 0.9257, + "rewards/accuracies": 1.0, + "rewards/chosen": -11.721475601196289, + "rewards/margins": 20.296268463134766, + "rewards/rejected": -32.01774597167969, + "step": 1130 + }, + { + "epoch": 0.8263114976895896, + "grad_norm": 2.3969268798828125, + "learning_rate": 6.775784314464717e-07, + "logits/chosen": -2.298384428024292, + "logits/rejected": -2.7226641178131104, + "logps/chosen": -8.508520126342773, + "logps/rejected": -21.121414184570312, + "loss": 0.5027, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -12.762779235839844, + "rewards/margins": 18.919343948364258, + "rewards/rejected": -31.6821231842041, + "step": 1140 + }, + { + "epoch": 0.8335598441605508, + "grad_norm": 1.3444470167160034, + "learning_rate": 6.421379363065142e-07, + "logits/chosen": -2.274933338165283, + "logits/rejected": -2.871828317642212, + "logps/chosen": -8.347254753112793, + "logps/rejected": -22.654220581054688, + "loss": 0.7009, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -12.520882606506348, + "rewards/margins": 21.46044921875, + "rewards/rejected": -33.98133087158203, + "step": 1150 + }, + { + "epoch": 0.8335598441605508, + "eval_logits/chosen": -2.3756191730499268, + "eval_logits/rejected": -2.7599036693573, + "eval_logps/chosen": -9.69633960723877, + "eval_logps/rejected": -20.757549285888672, + "eval_loss": 0.09489800035953522, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -14.544509887695312, + "eval_rewards/margins": 16.591815948486328, + "eval_rewards/rejected": -31.136323928833008, + "eval_runtime": 30.3194, + "eval_samples_per_second": 29.42, + "eval_steps_per_second": 3.694, + "step": 1150 + }, + { + "epoch": 0.8408081906315121, + "grad_norm": 3.957143783569336, + "learning_rate": 6.075123608706093e-07, + "logits/chosen": -2.2645936012268066, + "logits/rejected": -2.7766642570495605, + "logps/chosen": -9.488225936889648, + "logps/rejected": -20.858362197875977, + "loss": 0.9432, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -14.232337951660156, + "rewards/margins": 17.05520248413086, + "rewards/rejected": -31.287546157836914, + "step": 1160 + }, + { + "epoch": 0.8480565371024735, + "grad_norm": 0.2772391140460968, + "learning_rate": 5.737168930605272e-07, + "logits/chosen": -2.322751760482788, + "logits/rejected": -2.762363910675049, + "logps/chosen": -8.993582725524902, + "logps/rejected": -22.878589630126953, + "loss": 0.5761, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -13.490374565124512, + "rewards/margins": 20.8275089263916, + "rewards/rejected": -34.3178825378418, + "step": 1170 + }, + { + "epoch": 0.8553048835734348, + "grad_norm": 1.3643966913223267, + "learning_rate": 5.407663566854008e-07, + "logits/chosen": -2.365164279937744, + "logits/rejected": -2.7776386737823486, + "logps/chosen": -8.55947208404541, + "logps/rejected": -21.43111801147461, + "loss": 0.8883, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -12.839208602905273, + "rewards/margins": 19.30746841430664, + "rewards/rejected": -32.14667892456055, + "step": 1180 + }, + { + "epoch": 0.8625532300443961, + "grad_norm": 5.047004222869873, + "learning_rate": 5.086752049395094e-07, + "logits/chosen": -2.245209217071533, + "logits/rejected": -2.725189447402954, + "logps/chosen": -8.62891674041748, + "logps/rejected": -22.020822525024414, + "loss": 1.1177, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -12.943374633789062, + "rewards/margins": 20.087858200073242, + "rewards/rejected": -33.0312385559082, + "step": 1190 + }, + { + "epoch": 0.8698015765153574, + "grad_norm": 5.6105451583862305, + "learning_rate": 4.774575140626317e-07, + "logits/chosen": -2.2240893840789795, + "logits/rejected": -2.685760498046875, + "logps/chosen": -7.7927656173706055, + "logps/rejected": -21.007144927978516, + "loss": 0.4369, + "rewards/accuracies": 1.0, + "rewards/chosen": -11.68914794921875, + "rewards/margins": 19.821565628051758, + "rewards/rejected": -31.51071548461914, + "step": 1200 + }, + { + "epoch": 0.8698015765153574, + "eval_logits/chosen": -2.3455567359924316, + "eval_logits/rejected": -2.725982189178467, + "eval_logps/chosen": -8.64130973815918, + "eval_logps/rejected": -19.878324508666992, + "eval_loss": 0.08837977796792984, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -12.96196460723877, + "eval_rewards/margins": 16.855520248413086, + "eval_rewards/rejected": -29.817485809326172, + "eval_runtime": 30.3327, + "eval_samples_per_second": 29.407, + "eval_steps_per_second": 3.692, + "step": 1200 + }, + { + "epoch": 0.8770499229863188, + "grad_norm": 11.951034545898438, + "learning_rate": 4.4712697716573994e-07, + "logits/chosen": -2.388855457305908, + "logits/rejected": -2.7460148334503174, + "logps/chosen": -8.893162727355957, + "logps/rejected": -20.821142196655273, + "loss": 0.4923, + "rewards/accuracies": 1.0, + "rewards/chosen": -13.339744567871094, + "rewards/margins": 17.8919677734375, + "rewards/rejected": -31.231714248657227, + "step": 1210 + }, + { + "epoch": 0.8842982694572801, + "grad_norm": 1.99110746383667, + "learning_rate": 4.1769689822475147e-07, + "logits/chosen": -2.3649277687072754, + "logits/rejected": -2.76951265335083, + "logps/chosen": -9.380061149597168, + "logps/rejected": -22.044801712036133, + "loss": 0.7736, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -14.070094108581543, + "rewards/margins": 18.99711036682129, + "rewards/rejected": -33.06719970703125, + "step": 1220 + }, + { + "epoch": 0.8915466159282414, + "grad_norm": 3.7209253311157227, + "learning_rate": 3.891801862449629e-07, + "logits/chosen": -2.298079490661621, + "logits/rejected": -2.703709125518799, + "logps/chosen": -9.0845308303833, + "logps/rejected": -22.618410110473633, + "loss": 0.6428, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -13.626797676086426, + "rewards/margins": 20.300819396972656, + "rewards/rejected": -33.92761993408203, + "step": 1230 + }, + { + "epoch": 0.8987949623992026, + "grad_norm": 16.902679443359375, + "learning_rate": 3.615893495987335e-07, + "logits/chosen": -2.2431535720825195, + "logits/rejected": -2.703981876373291, + "logps/chosen": -8.514019966125488, + "logps/rejected": -20.308120727539062, + "loss": 0.5315, + "rewards/accuracies": 1.0, + "rewards/chosen": -12.771028518676758, + "rewards/margins": 17.691150665283203, + "rewards/rejected": -30.46217918395996, + "step": 1240 + }, + { + "epoch": 0.906043308870164, + "grad_norm": 3.4878218173980713, + "learning_rate": 3.3493649053890325e-07, + "logits/chosen": -2.2696893215179443, + "logits/rejected": -2.74363112449646, + "logps/chosen": -8.124156951904297, + "logps/rejected": -20.82503318786621, + "loss": 0.9809, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -12.186236381530762, + "rewards/margins": 19.05131721496582, + "rewards/rejected": -31.237552642822266, + "step": 1250 + }, + { + "epoch": 0.906043308870164, + "eval_logits/chosen": -2.3479084968566895, + "eval_logits/rejected": -2.7300288677215576, + "eval_logps/chosen": -8.8316068649292, + "eval_logps/rejected": -20.414066314697266, + "eval_loss": 0.08777090907096863, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -13.247411727905273, + "eval_rewards/margins": 17.373689651489258, + "eval_rewards/rejected": -30.621097564697266, + "eval_runtime": 30.3197, + "eval_samples_per_second": 29.42, + "eval_steps_per_second": 3.694, + "step": 1250 + }, + { + "epoch": 0.9132916553411253, + "grad_norm": 3.848902463912964, + "learning_rate": 3.092332998903416e-07, + "logits/chosen": -2.3060476779937744, + "logits/rejected": -2.790208101272583, + "logps/chosen": -7.368963718414307, + "logps/rejected": -19.993572235107422, + "loss": 1.1088, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -11.053446769714355, + "rewards/margins": 18.936914443969727, + "rewards/rejected": -29.990360260009766, + "step": 1260 + }, + { + "epoch": 0.9205400018120866, + "grad_norm": 13.889841079711914, + "learning_rate": 2.844910519219632e-07, + "logits/chosen": -2.33646821975708, + "logits/rejected": -2.6650798320770264, + "logps/chosen": -8.992429733276367, + "logps/rejected": -20.459260940551758, + "loss": 0.8384, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -13.48864459991455, + "rewards/margins": 17.200244903564453, + "rewards/rejected": -30.688892364501953, + "step": 1270 + }, + { + "epoch": 0.9277883482830479, + "grad_norm": 2.3417980670928955, + "learning_rate": 2.6072059940146775e-07, + "logits/chosen": -2.341177225112915, + "logits/rejected": -2.7101569175720215, + "logps/chosen": -8.923599243164062, + "logps/rejected": -21.028385162353516, + "loss": 0.6228, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -13.385398864746094, + "rewards/margins": 18.157176971435547, + "rewards/rejected": -31.54257583618164, + "step": 1280 + }, + { + "epoch": 0.9350366947540092, + "grad_norm": 9.856499671936035, + "learning_rate": 2.3793236883495164e-07, + "logits/chosen": -2.3007121086120605, + "logits/rejected": -2.7542452812194824, + "logps/chosen": -8.09909439086914, + "logps/rejected": -21.306743621826172, + "loss": 1.1599, + "rewards/accuracies": 1.0, + "rewards/chosen": -12.148639678955078, + "rewards/margins": 19.811477661132812, + "rewards/rejected": -31.960119247436523, + "step": 1290 + }, + { + "epoch": 0.9422850412249706, + "grad_norm": 2.1335718631744385, + "learning_rate": 2.1613635589349756e-07, + "logits/chosen": -2.2931392192840576, + "logits/rejected": -2.709573984146118, + "logps/chosen": -8.620733261108398, + "logps/rejected": -20.675912857055664, + "loss": 0.5808, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -12.931098937988281, + "rewards/margins": 18.082773208618164, + "rewards/rejected": -31.013874053955078, + "step": 1300 + }, + { + "epoch": 0.9422850412249706, + "eval_logits/chosen": -2.343172550201416, + "eval_logits/rejected": -2.723876476287842, + "eval_logps/chosen": -8.513604164123535, + "eval_logps/rejected": -20.223154067993164, + "eval_loss": 0.09216763824224472, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -12.770406723022461, + "eval_rewards/margins": 17.56432342529297, + "eval_rewards/rejected": -30.33473014831543, + "eval_runtime": 30.338, + "eval_samples_per_second": 29.402, + "eval_steps_per_second": 3.692, + "step": 1300 + }, + { + "epoch": 0.9495333876959319, + "grad_norm": 9.816523551940918, + "learning_rate": 1.95342121028749e-07, + "logits/chosen": -2.3087425231933594, + "logits/rejected": -2.7727978229522705, + "logps/chosen": -7.882437229156494, + "logps/rejected": -21.284303665161133, + "loss": 0.8332, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -11.823655128479004, + "rewards/margins": 20.102800369262695, + "rewards/rejected": -31.926456451416016, + "step": 1310 + }, + { + "epoch": 0.9567817341668932, + "grad_norm": 4.905272960662842, + "learning_rate": 1.7555878527937164e-07, + "logits/chosen": -2.3210203647613525, + "logits/rejected": -2.7130990028381348, + "logps/chosen": -8.509033203125, + "logps/rejected": -20.50202751159668, + "loss": 0.642, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -12.763550758361816, + "rewards/margins": 17.989490509033203, + "rewards/rejected": -30.753040313720703, + "step": 1320 + }, + { + "epoch": 0.9640300806378544, + "grad_norm": 3.753002166748047, + "learning_rate": 1.567950262702714e-07, + "logits/chosen": -2.3174471855163574, + "logits/rejected": -2.7005510330200195, + "logps/chosen": -7.850353240966797, + "logps/rejected": -20.87979507446289, + "loss": 0.4295, + "rewards/accuracies": 1.0, + "rewards/chosen": -11.775530815124512, + "rewards/margins": 19.54416275024414, + "rewards/rejected": -31.319690704345703, + "step": 1330 + }, + { + "epoch": 0.9712784271088158, + "grad_norm": 0.9552884697914124, + "learning_rate": 1.3905907440629752e-07, + "logits/chosen": -2.259413242340088, + "logits/rejected": -2.7716660499572754, + "logps/chosen": -8.414698600769043, + "logps/rejected": -22.918161392211914, + "loss": 1.3939, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -12.62204647064209, + "rewards/margins": 21.75519561767578, + "rewards/rejected": -34.37724304199219, + "step": 1340 + }, + { + "epoch": 0.9785267735797771, + "grad_norm": 9.898683547973633, + "learning_rate": 1.223587092621162e-07, + "logits/chosen": -2.2864937782287598, + "logits/rejected": -2.8085687160491943, + "logps/chosen": -8.638604164123535, + "logps/rejected": -22.094703674316406, + "loss": 0.7096, + "rewards/accuracies": 1.0, + "rewards/chosen": -12.957905769348145, + "rewards/margins": 20.18414878845215, + "rewards/rejected": -33.142051696777344, + "step": 1350 + }, + { + "epoch": 0.9785267735797771, + "eval_logits/chosen": -2.352558135986328, + "eval_logits/rejected": -2.736541271209717, + "eval_logps/chosen": -9.229049682617188, + "eval_logps/rejected": -20.852691650390625, + "eval_loss": 0.08608455955982208, + "eval_rewards/accuracies": 0.9642857313156128, + "eval_rewards/chosen": -13.843575477600098, + "eval_rewards/margins": 17.435461044311523, + "eval_rewards/rejected": -31.279041290283203, + "eval_runtime": 30.3501, + "eval_samples_per_second": 29.39, + "eval_steps_per_second": 3.69, + "step": 1350 + }, + { + "epoch": 0.9857751200507384, + "grad_norm": 32.38576889038086, + "learning_rate": 1.067012561698319e-07, + "logits/chosen": -2.235184669494629, + "logits/rejected": -2.7410881519317627, + "logps/chosen": -7.7364630699157715, + "logps/rejected": -20.32790184020996, + "loss": 0.6009, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -11.604695320129395, + "rewards/margins": 18.887157440185547, + "rewards/rejected": -30.491851806640625, + "step": 1360 + }, + { + "epoch": 0.9930234665216997, + "grad_norm": 2.9375524520874023, + "learning_rate": 9.209358300585474e-08, + "logits/chosen": -2.37656831741333, + "logits/rejected": -2.6760752201080322, + "logps/chosen": -9.913487434387207, + "logps/rejected": -21.843143463134766, + "loss": 0.6913, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -14.870233535766602, + "rewards/margins": 17.894481658935547, + "rewards/rejected": -32.764713287353516, + "step": 1370 + }, + { + "epoch": 1.0, + "grad_norm": 6.210167407989502, + "learning_rate": 7.854209717842231e-08, + "logits/chosen": -2.3600404262542725, + "logits/rejected": -2.8249471187591553, + "logps/chosen": -8.988312721252441, + "logps/rejected": -22.236913681030273, + "loss": 0.3924, + "rewards/accuracies": 1.0, + "rewards/chosen": -13.482467651367188, + "rewards/margins": 19.872901916503906, + "rewards/rejected": -33.35537338256836, + "step": 1380 + }, + { + "epoch": 1.0072483464709614, + "grad_norm": 4.497740745544434, + "learning_rate": 6.605274281709929e-08, + "logits/chosen": -2.3760697841644287, + "logits/rejected": -2.748159646987915, + "logps/chosen": -9.209830284118652, + "logps/rejected": -20.537687301635742, + "loss": 1.4012, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -13.814745903015137, + "rewards/margins": 16.99178695678711, + "rewards/rejected": -30.806529998779297, + "step": 1390 + }, + { + "epoch": 1.0144966929419226, + "grad_norm": 3.305466651916504, + "learning_rate": 5.463099816548578e-08, + "logits/chosen": -2.3499505519866943, + "logits/rejected": -2.745049238204956, + "logps/chosen": -9.211877822875977, + "logps/rejected": -21.599735260009766, + "loss": 0.6699, + "rewards/accuracies": 0.9375, + "rewards/chosen": -13.817817687988281, + "rewards/margins": 18.581789016723633, + "rewards/rejected": -32.39960479736328, + "step": 1400 + }, + { + "epoch": 1.0144966929419226, + "eval_logits/chosen": -2.3516786098480225, + "eval_logits/rejected": -2.734497308731079, + "eval_logps/chosen": -9.099111557006836, + "eval_logps/rejected": -20.884517669677734, + "eval_loss": 0.08683376014232635, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -13.64866828918457, + "eval_rewards/margins": 17.6781063079834, + "eval_rewards/rejected": -31.326778411865234, + "eval_runtime": 30.3264, + "eval_samples_per_second": 29.413, + "eval_steps_per_second": 3.693, + "step": 1400 + }, + { + "epoch": 1.021745039412884, + "grad_norm": 17.777156829833984, + "learning_rate": 4.428187317827848e-08, + "logits/chosen": -2.3576529026031494, + "logits/rejected": -2.791233777999878, + "logps/chosen": -8.674802780151367, + "logps/rejected": -21.15481948852539, + "loss": 0.7684, + "rewards/accuracies": 1.0, + "rewards/chosen": -13.01220417022705, + "rewards/margins": 18.72002410888672, + "rewards/rejected": -31.732227325439453, + "step": 1410 + }, + { + "epoch": 1.0289933858838451, + "grad_norm": 2.346339225769043, + "learning_rate": 3.5009907323737826e-08, + "logits/chosen": -2.365281581878662, + "logits/rejected": -2.7309329509735107, + "logps/chosen": -8.632737159729004, + "logps/rejected": -21.099023818969727, + "loss": 0.6473, + "rewards/accuracies": 1.0, + "rewards/chosen": -12.949106216430664, + "rewards/margins": 18.699430465698242, + "rewards/rejected": -31.648534774780273, + "step": 1420 + }, + { + "epoch": 1.0362417323548065, + "grad_norm": 2.2487072944641113, + "learning_rate": 2.681916759252917e-08, + "logits/chosen": -2.356719970703125, + "logits/rejected": -2.8153066635131836, + "logps/chosen": -8.822325706481934, + "logps/rejected": -22.613697052001953, + "loss": 0.5875, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -13.233488082885742, + "rewards/margins": 20.687055587768555, + "rewards/rejected": -33.9205436706543, + "step": 1430 + }, + { + "epoch": 1.043490078825768, + "grad_norm": 1.1723434925079346, + "learning_rate": 1.9713246713805588e-08, + "logits/chosen": -2.255270004272461, + "logits/rejected": -2.697756290435791, + "logps/chosen": -8.767889022827148, + "logps/rejected": -21.596534729003906, + "loss": 0.7658, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -13.151834487915039, + "rewards/margins": 19.24296760559082, + "rewards/rejected": -32.394798278808594, + "step": 1440 + }, + { + "epoch": 1.0507384252967291, + "grad_norm": 2.121746063232422, + "learning_rate": 1.3695261579316776e-08, + "logits/chosen": -2.330730676651001, + "logits/rejected": -2.73622727394104, + "logps/chosen": -8.807150840759277, + "logps/rejected": -20.8906192779541, + "loss": 0.6331, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -13.210726737976074, + "rewards/margins": 18.12520408630371, + "rewards/rejected": -31.335927963256836, + "step": 1450 + }, + { + "epoch": 1.0507384252967291, + "eval_logits/chosen": -2.349879503250122, + "eval_logits/rejected": -2.732649326324463, + "eval_logps/chosen": -9.049729347229004, + "eval_logps/rejected": -20.825775146484375, + "eval_loss": 0.08744800090789795, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -13.574592590332031, + "eval_rewards/margins": 17.6640682220459, + "eval_rewards/rejected": -31.238664627075195, + "eval_runtime": 31.2343, + "eval_samples_per_second": 28.558, + "eval_steps_per_second": 3.586, + "step": 1450 + }, + { + "epoch": 1.0579867717676905, + "grad_norm": 8.632430076599121, + "learning_rate": 8.767851876239075e-09, + "logits/chosen": -2.3430233001708984, + "logits/rejected": -2.801687717437744, + "logps/chosen": -9.03283405303955, + "logps/rejected": -22.0975341796875, + "loss": 0.709, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -13.549250602722168, + "rewards/margins": 19.597049713134766, + "rewards/rejected": -33.14630126953125, + "step": 1460 + }, + { + "epoch": 1.0652351182386517, + "grad_norm": 1.3894760608673096, + "learning_rate": 4.933178929321103e-09, + "logits/chosen": -2.3081536293029785, + "logits/rejected": -2.7821733951568604, + "logps/chosen": -8.711824417114258, + "logps/rejected": -22.871994018554688, + "loss": 0.5232, + "rewards/accuracies": 1.0, + "rewards/chosen": -13.06773567199707, + "rewards/margins": 21.240253448486328, + "rewards/rejected": -34.30799102783203, + "step": 1470 + }, + { + "epoch": 1.072483464709613, + "grad_norm": 1.5356446504592896, + "learning_rate": 2.192924752854042e-09, + "logits/chosen": -2.3169848918914795, + "logits/rejected": -2.7624945640563965, + "logps/chosen": -9.128572463989258, + "logps/rejected": -21.637937545776367, + "loss": 0.7016, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -13.692858695983887, + "rewards/margins": 18.7640438079834, + "rewards/rejected": -32.45690155029297, + "step": 1480 + }, + { + "epoch": 1.0797318111805745, + "grad_norm": 7.452216625213623, + "learning_rate": 5.48291312886251e-10, + "logits/chosen": -2.2697954177856445, + "logits/rejected": -2.696985960006714, + "logps/chosen": -9.422815322875977, + "logps/rejected": -22.212636947631836, + "loss": 0.454, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -14.134222030639648, + "rewards/margins": 19.184734344482422, + "rewards/rejected": -33.3189582824707, + "step": 1490 + }, + { + "epoch": 1.0869801576515357, + "grad_norm": 5.824347972869873, + "learning_rate": 0.0, + "logits/chosen": -2.3115651607513428, + "logits/rejected": -2.7756543159484863, + "logps/chosen": -8.15144157409668, + "logps/rejected": -21.585973739624023, + "loss": 0.3791, + "rewards/accuracies": 1.0, + "rewards/chosen": -12.227163314819336, + "rewards/margins": 20.151796340942383, + "rewards/rejected": -32.37895965576172, + "step": 1500 + }, + { + "epoch": 1.0869801576515357, + "eval_logits/chosen": -2.3500683307647705, + "eval_logits/rejected": -2.7328763008117676, + "eval_logps/chosen": -9.082016944885254, + "eval_logps/rejected": -20.82696533203125, + "eval_loss": 0.08840522915124893, + "eval_rewards/accuracies": 0.9553571343421936, + "eval_rewards/chosen": -13.623023986816406, + "eval_rewards/margins": 17.617422103881836, + "eval_rewards/rejected": -31.240449905395508, + "eval_runtime": 30.3063, + "eval_samples_per_second": 29.433, + "eval_steps_per_second": 3.696, + "step": 1500 + } + ], + "logging_steps": 10, + "max_steps": 1500, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.57632267653795e+18, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}