diff --git "a/checkpoint-1500/trainer_state.json" "b/checkpoint-1500/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-1500/trainer_state.json" @@ -0,0 +1,2763 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.3321492007104796, + "eval_steps": 50, + "global_step": 1500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.008880994671403197, + "grad_norm": 0.04571289196610451, + "learning_rate": 4.999451708687114e-06, + "logits/chosen": 14.56671142578125, + "logits/rejected": 15.112574577331543, + "logps/chosen": -0.26506316661834717, + "logps/rejected": -0.3439488410949707, + "loss": 0.9267, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.39759472012519836, + "rewards/margins": 0.11832849681377411, + "rewards/rejected": -0.5159232020378113, + "step": 10 + }, + { + "epoch": 0.017761989342806393, + "grad_norm": 0.0512714721262455, + "learning_rate": 4.997807075247147e-06, + "logits/chosen": 14.376543045043945, + "logits/rejected": 14.862703323364258, + "logps/chosen": -0.2708089351654053, + "logps/rejected": -0.32412824034690857, + "loss": 0.936, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4062133729457855, + "rewards/margins": 0.07997899502515793, + "rewards/rejected": -0.4861923158168793, + "step": 20 + }, + { + "epoch": 0.02664298401420959, + "grad_norm": 0.058383647352457047, + "learning_rate": 4.9950668210706795e-06, + "logits/chosen": 14.208717346191406, + "logits/rejected": 15.370651245117188, + "logps/chosen": -0.28206294775009155, + "logps/rejected": -0.38387423753738403, + "loss": 0.9215, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.42309442162513733, + "rewards/margins": 0.15271687507629395, + "rewards/rejected": -0.5758112668991089, + "step": 30 + }, + { + "epoch": 0.035523978685612786, + "grad_norm": 0.06262075155973434, + "learning_rate": 4.9912321481237616e-06, + "logits/chosen": 14.768765449523926, + "logits/rejected": 15.169331550598145, + "logps/chosen": -0.27857059240341187, + "logps/rejected": -0.3388269543647766, + "loss": 0.9386, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4178559184074402, + "rewards/margins": 0.09038447588682175, + "rewards/rejected": -0.5082404017448425, + "step": 40 + }, + { + "epoch": 0.04440497335701599, + "grad_norm": 0.06259036809206009, + "learning_rate": 4.986304738420684e-06, + "logits/chosen": 14.950456619262695, + "logits/rejected": 15.232122421264648, + "logps/chosen": -0.2961367070674896, + "logps/rejected": -0.3322262465953827, + "loss": 0.9317, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.44420504570007324, + "rewards/margins": 0.054134320467710495, + "rewards/rejected": -0.4983394145965576, + "step": 50 + }, + { + "epoch": 0.04440497335701599, + "eval_logits/chosen": 14.56529426574707, + "eval_logits/rejected": 14.895020484924316, + "eval_logps/chosen": -0.2806546986103058, + "eval_logps/rejected": -0.3486972451210022, + "eval_loss": 0.9381324052810669, + "eval_rewards/accuracies": 0.5274725556373596, + "eval_rewards/chosen": -0.4209820330142975, + "eval_rewards/margins": 0.10206379741430283, + "eval_rewards/rejected": -0.5230458974838257, + "eval_runtime": 25.2574, + "eval_samples_per_second": 28.823, + "eval_steps_per_second": 3.603, + "step": 50 + }, + { + "epoch": 0.05328596802841918, + "grad_norm": 0.07301533967256546, + "learning_rate": 4.980286753286196e-06, + "logits/chosen": 14.195574760437012, + "logits/rejected": 15.173194885253906, + "logps/chosen": -0.2693648636341095, + "logps/rejected": -0.33997970819473267, + "loss": 0.9319, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -0.40404725074768066, + "rewards/margins": 0.10592226684093475, + "rewards/rejected": -0.5099694728851318, + "step": 60 + }, + { + "epoch": 0.06216696269982238, + "grad_norm": 0.0659889206290245, + "learning_rate": 4.973180832407471e-06, + "logits/chosen": 14.910173416137695, + "logits/rejected": 15.361429214477539, + "logps/chosen": -0.28456225991249084, + "logps/rejected": -0.3702812194824219, + "loss": 0.9185, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.42684346437454224, + "rewards/margins": 0.12857840955257416, + "rewards/rejected": -0.5554218292236328, + "step": 70 + }, + { + "epoch": 0.07104795737122557, + "grad_norm": 0.05815625935792923, + "learning_rate": 4.964990092676263e-06, + "logits/chosen": 14.407182693481445, + "logits/rejected": 14.948204040527344, + "logps/chosen": -0.292889267206192, + "logps/rejected": -0.3381648063659668, + "loss": 0.9388, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -0.43933385610580444, + "rewards/margins": 0.06791339069604874, + "rewards/rejected": -0.5072472095489502, + "step": 80 + }, + { + "epoch": 0.07992895204262877, + "grad_norm": 0.06627190113067627, + "learning_rate": 4.9557181268217225e-06, + "logits/chosen": 14.622471809387207, + "logits/rejected": 15.167770385742188, + "logps/chosen": -0.28155821561813354, + "logps/rejected": -0.33633899688720703, + "loss": 0.9256, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -0.4223373532295227, + "rewards/margins": 0.08217118680477142, + "rewards/rejected": -0.5045084953308105, + "step": 90 + }, + { + "epoch": 0.08880994671403197, + "grad_norm": 0.0724545568227768, + "learning_rate": 4.9453690018345144e-06, + "logits/chosen": 14.289724349975586, + "logits/rejected": 14.882037162780762, + "logps/chosen": -0.2791440486907959, + "logps/rejected": -0.35329627990722656, + "loss": 0.9374, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -0.41871610283851624, + "rewards/margins": 0.11122839152812958, + "rewards/rejected": -0.5299445390701294, + "step": 100 + }, + { + "epoch": 0.08880994671403197, + "eval_logits/chosen": 14.337930679321289, + "eval_logits/rejected": 14.689269065856934, + "eval_logps/chosen": -0.2726942300796509, + "eval_logps/rejected": -0.34668418765068054, + "eval_loss": 0.9302808046340942, + "eval_rewards/accuracies": 0.5384615659713745, + "eval_rewards/chosen": -0.40904131531715393, + "eval_rewards/margins": 0.11098497360944748, + "eval_rewards/rejected": -0.5200263261795044, + "eval_runtime": 25.2585, + "eval_samples_per_second": 28.822, + "eval_steps_per_second": 3.603, + "step": 100 + }, + { + "epoch": 0.09769094138543517, + "grad_norm": 0.08156246691942215, + "learning_rate": 4.933947257182901e-06, + "logits/chosen": 14.499124526977539, + "logits/rejected": 14.916313171386719, + "logps/chosen": -0.2798352837562561, + "logps/rejected": -0.3477734327316284, + "loss": 0.9243, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4197530150413513, + "rewards/margins": 0.10190720856189728, + "rewards/rejected": -0.5216602087020874, + "step": 110 + }, + { + "epoch": 0.10657193605683836, + "grad_norm": 0.08161844313144684, + "learning_rate": 4.921457902821578e-06, + "logits/chosen": 13.595013618469238, + "logits/rejected": 14.390353202819824, + "logps/chosen": -0.26682502031326294, + "logps/rejected": -0.3336995542049408, + "loss": 0.9123, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.400237500667572, + "rewards/margins": 0.10031183809041977, + "rewards/rejected": -0.5005493760108948, + "step": 120 + }, + { + "epoch": 0.11545293072824156, + "grad_norm": 0.28624778985977173, + "learning_rate": 4.907906416994146e-06, + "logits/chosen": 13.711044311523438, + "logits/rejected": 14.558542251586914, + "logps/chosen": -0.27874043583869934, + "logps/rejected": -0.3582325279712677, + "loss": 0.9163, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -0.41811060905456543, + "rewards/margins": 0.11923813819885254, + "rewards/rejected": -0.537348747253418, + "step": 130 + }, + { + "epoch": 0.12433392539964476, + "grad_norm": 0.10971464216709137, + "learning_rate": 4.893298743830168e-06, + "logits/chosen": 14.18798828125, + "logits/rejected": 14.993026733398438, + "logps/chosen": -0.2750400900840759, + "logps/rejected": -0.39451608061790466, + "loss": 0.9098, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.4125601351261139, + "rewards/margins": 0.17921395599842072, + "rewards/rejected": -0.5917741060256958, + "step": 140 + }, + { + "epoch": 0.13321492007104796, + "grad_norm": 0.09321591258049011, + "learning_rate": 4.8776412907378845e-06, + "logits/chosen": 12.775139808654785, + "logits/rejected": 13.751996994018555, + "logps/chosen": -0.28446996212005615, + "logps/rejected": -0.36404967308044434, + "loss": 0.9104, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.42670494318008423, + "rewards/margins": 0.11936960369348526, + "rewards/rejected": -0.5460745096206665, + "step": 150 + }, + { + "epoch": 0.13321492007104796, + "eval_logits/chosen": 12.97266960144043, + "eval_logits/rejected": 13.47339916229248, + "eval_logps/chosen": -0.27297571301460266, + "eval_logps/rejected": -0.36854612827301025, + "eval_loss": 0.9143257737159729, + "eval_rewards/accuracies": 0.5824176073074341, + "eval_rewards/chosen": -0.4094635546207428, + "eval_rewards/margins": 0.14335563778877258, + "eval_rewards/rejected": -0.5528191924095154, + "eval_runtime": 25.2406, + "eval_samples_per_second": 28.842, + "eval_steps_per_second": 3.605, + "step": 150 + }, + { + "epoch": 0.14209591474245115, + "grad_norm": 0.11029861867427826, + "learning_rate": 4.860940925593703e-06, + "logits/chosen": 12.677947998046875, + "logits/rejected": 13.396716117858887, + "logps/chosen": -0.2631794512271881, + "logps/rejected": -0.37102141976356506, + "loss": 0.9051, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.3947691321372986, + "rewards/margins": 0.161762997508049, + "rewards/rejected": -0.5565321445465088, + "step": 160 + }, + { + "epoch": 0.15097690941385436, + "grad_norm": 0.15728294849395752, + "learning_rate": 4.84320497372973e-06, + "logits/chosen": 12.620219230651855, + "logits/rejected": 13.189640998840332, + "logps/chosen": -0.2947639524936676, + "logps/rejected": -0.3843482732772827, + "loss": 0.8906, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.442145973443985, + "rewards/margins": 0.13437646627426147, + "rewards/rejected": -0.5765224099159241, + "step": 170 + }, + { + "epoch": 0.15985790408525755, + "grad_norm": 0.31504154205322266, + "learning_rate": 4.824441214720629e-06, + "logits/chosen": 11.487619400024414, + "logits/rejected": 12.33470344543457, + "logps/chosen": -0.271095871925354, + "logps/rejected": -0.4252637028694153, + "loss": 0.8766, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.406643807888031, + "rewards/margins": 0.2312517911195755, + "rewards/rejected": -0.6378955245018005, + "step": 180 + }, + { + "epoch": 0.16873889875666073, + "grad_norm": 0.19222252070903778, + "learning_rate": 4.804657878971252e-06, + "logits/chosen": 10.093737602233887, + "logits/rejected": 10.851752281188965, + "logps/chosen": -0.2679918110370636, + "logps/rejected": -0.437336266040802, + "loss": 0.884, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.4019877314567566, + "rewards/margins": 0.2540166974067688, + "rewards/rejected": -0.6560044288635254, + "step": 190 + }, + { + "epoch": 0.17761989342806395, + "grad_norm": 0.2275688648223877, + "learning_rate": 4.783863644106502e-06, + "logits/chosen": 9.483477592468262, + "logits/rejected": 10.106366157531738, + "logps/chosen": -0.2957404553890228, + "logps/rejected": -0.40739065408706665, + "loss": 0.8767, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.4436107575893402, + "rewards/margins": 0.16747523844242096, + "rewards/rejected": -0.6110859513282776, + "step": 200 + }, + { + "epoch": 0.17761989342806395, + "eval_logits/chosen": 8.491498947143555, + "eval_logits/rejected": 8.999146461486816, + "eval_logps/chosen": -0.3135836124420166, + "eval_logps/rejected": -0.4829566180706024, + "eval_loss": 0.8664290904998779, + "eval_rewards/accuracies": 0.6263736486434937, + "eval_rewards/chosen": -0.4703753888607025, + "eval_rewards/margins": 0.2540595233440399, + "eval_rewards/rejected": -0.7244349122047424, + "eval_runtime": 25.2553, + "eval_samples_per_second": 28.826, + "eval_steps_per_second": 3.603, + "step": 200 + }, + { + "epoch": 0.18650088809946713, + "grad_norm": 0.27885496616363525, + "learning_rate": 4.762067631165049e-06, + "logits/chosen": 7.234966278076172, + "logits/rejected": 8.313450813293457, + "logps/chosen": -0.29102542996406555, + "logps/rejected": -0.49241799116134644, + "loss": 0.8556, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.43653813004493713, + "rewards/margins": 0.3020888566970825, + "rewards/rejected": -0.7386269569396973, + "step": 210 + }, + { + "epoch": 0.19538188277087035, + "grad_norm": 0.29907363653182983, + "learning_rate": 4.7392794005985324e-06, + "logits/chosen": 7.907521724700928, + "logits/rejected": 8.253190994262695, + "logps/chosen": -0.33691853284835815, + "logps/rejected": -0.4829257130622864, + "loss": 0.8236, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.5053777694702148, + "rewards/margins": 0.21901080012321472, + "rewards/rejected": -0.724388599395752, + "step": 220 + }, + { + "epoch": 0.20426287744227353, + "grad_norm": 0.282474547624588, + "learning_rate": 4.715508948078037e-06, + "logits/chosen": 6.367492198944092, + "logits/rejected": 6.273728370666504, + "logps/chosen": -0.3519875705242157, + "logps/rejected": -0.5284813642501831, + "loss": 0.8027, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5279813408851624, + "rewards/margins": 0.2647407650947571, + "rewards/rejected": -0.7927221059799194, + "step": 230 + }, + { + "epoch": 0.21314387211367672, + "grad_norm": 0.327765554189682, + "learning_rate": 4.690766700109659e-06, + "logits/chosen": 5.090893268585205, + "logits/rejected": 4.768380165100098, + "logps/chosen": -0.3851698040962219, + "logps/rejected": -0.6464222073554993, + "loss": 0.7898, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.5777546167373657, + "rewards/margins": 0.391878604888916, + "rewards/rejected": -0.9696332812309265, + "step": 240 + }, + { + "epoch": 0.22202486678507993, + "grad_norm": 0.4895865321159363, + "learning_rate": 4.665063509461098e-06, + "logits/chosen": 4.056812286376953, + "logits/rejected": 3.723601818084717, + "logps/chosen": -0.4400455951690674, + "logps/rejected": -0.7731422781944275, + "loss": 0.7626, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.6600683927536011, + "rewards/margins": 0.49964505434036255, + "rewards/rejected": -1.1597135066986084, + "step": 250 + }, + { + "epoch": 0.22202486678507993, + "eval_logits/chosen": 2.420060396194458, + "eval_logits/rejected": 2.1626052856445312, + "eval_logps/chosen": -0.4724067151546478, + "eval_logps/rejected": -0.8418064117431641, + "eval_loss": 0.7631083130836487, + "eval_rewards/accuracies": 0.6483516693115234, + "eval_rewards/chosen": -0.7086100578308105, + "eval_rewards/margins": 0.5540997385978699, + "eval_rewards/rejected": -1.2627097368240356, + "eval_runtime": 25.2418, + "eval_samples_per_second": 28.841, + "eval_steps_per_second": 3.605, + "step": 250 + }, + { + "epoch": 0.23090586145648312, + "grad_norm": 0.46291017532348633, + "learning_rate": 4.638410650401267e-06, + "logits/chosen": 1.5297390222549438, + "logits/rejected": 1.1381648778915405, + "logps/chosen": -0.4418027997016907, + "logps/rejected": -1.0542564392089844, + "loss": 0.7026, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.6627041697502136, + "rewards/margins": 0.9186803698539734, + "rewards/rejected": -1.5813844203948975, + "step": 260 + }, + { + "epoch": 0.23978685612788633, + "grad_norm": 0.9783313870429993, + "learning_rate": 4.610819813755038e-06, + "logits/chosen": 2.8311033248901367, + "logits/rejected": 1.9742711782455444, + "logps/chosen": -0.5430587530136108, + "logps/rejected": -0.9841039776802063, + "loss": 0.7317, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.814588189125061, + "rewards/margins": 0.6615679860115051, + "rewards/rejected": -1.4761559963226318, + "step": 270 + }, + { + "epoch": 0.24866785079928952, + "grad_norm": 2.102562189102173, + "learning_rate": 4.582303101775249e-06, + "logits/chosen": 1.8241952657699585, + "logits/rejected": 0.8777934312820435, + "logps/chosen": -0.5624039769172668, + "logps/rejected": -1.1460126638412476, + "loss": 0.6887, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.8436058163642883, + "rewards/margins": 0.8754131197929382, + "rewards/rejected": -1.7190189361572266, + "step": 280 + }, + { + "epoch": 0.25754884547069273, + "grad_norm": 0.9813026189804077, + "learning_rate": 4.55287302283426e-06, + "logits/chosen": 2.370732069015503, + "logits/rejected": 1.4697134494781494, + "logps/chosen": -0.6739786863327026, + "logps/rejected": -1.6581566333770752, + "loss": 0.5695, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.0109679698944092, + "rewards/margins": 1.476266622543335, + "rewards/rejected": -2.487234592437744, + "step": 290 + }, + { + "epoch": 0.2664298401420959, + "grad_norm": 2.187314510345459, + "learning_rate": 4.522542485937369e-06, + "logits/chosen": 1.6230781078338623, + "logits/rejected": 0.5460122227668762, + "logps/chosen": -0.6433733701705933, + "logps/rejected": -2.1001811027526855, + "loss": 0.5366, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.9650601148605347, + "rewards/margins": 2.185211658477783, + "rewards/rejected": -3.1502718925476074, + "step": 300 + }, + { + "epoch": 0.2664298401420959, + "eval_logits/chosen": 1.4087599515914917, + "eval_logits/rejected": 0.7888947129249573, + "eval_logps/chosen": -0.7579545974731445, + "eval_logps/rejected": -2.0049116611480713, + "eval_loss": 0.551510214805603, + "eval_rewards/accuracies": 0.6813187003135681, + "eval_rewards/chosen": -1.1369318962097168, + "eval_rewards/margins": 1.8704355955123901, + "eval_rewards/rejected": -3.0073673725128174, + "eval_runtime": 25.2647, + "eval_samples_per_second": 28.815, + "eval_steps_per_second": 3.602, + "step": 300 + }, + { + "epoch": 0.2753108348134991, + "grad_norm": 0.7035408616065979, + "learning_rate": 4.491324795060491e-06, + "logits/chosen": 1.5831315517425537, + "logits/rejected": 0.46730250120162964, + "logps/chosen": -0.7262418866157532, + "logps/rejected": -2.1209158897399902, + "loss": 0.5524, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.0893628597259521, + "rewards/margins": 2.092010974884033, + "rewards/rejected": -3.1813735961914062, + "step": 310 + }, + { + "epoch": 0.2841918294849023, + "grad_norm": 0.5678634643554688, + "learning_rate": 4.4592336433146e-06, + "logits/chosen": 1.265734076499939, + "logits/rejected": 0.7576489448547363, + "logps/chosen": -0.7938942313194275, + "logps/rejected": -2.3495612144470215, + "loss": 0.5233, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.1908413171768188, + "rewards/margins": 2.333500385284424, + "rewards/rejected": -3.5243420600891113, + "step": 320 + }, + { + "epoch": 0.29307282415630553, + "grad_norm": 1.1373224258422852, + "learning_rate": 4.426283106939474e-06, + "logits/chosen": 2.977414846420288, + "logits/rejected": 2.1573710441589355, + "logps/chosen": -0.8513160943984985, + "logps/rejected": -2.4125566482543945, + "loss": 0.556, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.2769742012023926, + "rewards/margins": 2.341860771179199, + "rewards/rejected": -3.6188347339630127, + "step": 330 + }, + { + "epoch": 0.3019538188277087, + "grad_norm": 4.7876176834106445, + "learning_rate": 4.3924876391293915e-06, + "logits/chosen": 2.4026589393615723, + "logits/rejected": 1.207395315170288, + "logps/chosen": -0.8529679179191589, + "logps/rejected": -2.456879138946533, + "loss": 0.5592, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.279451847076416, + "rewards/margins": 2.4058666229248047, + "rewards/rejected": -3.6853187084198, + "step": 340 + }, + { + "epoch": 0.3108348134991119, + "grad_norm": 0.5053763389587402, + "learning_rate": 4.357862063693486e-06, + "logits/chosen": 2.434265375137329, + "logits/rejected": 1.2504141330718994, + "logps/chosen": -0.9489291310310364, + "logps/rejected": -2.8521530628204346, + "loss": 0.4737, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.423393726348877, + "rewards/margins": 2.8548355102539062, + "rewards/rejected": -4.278229236602783, + "step": 350 + }, + { + "epoch": 0.3108348134991119, + "eval_logits/chosen": 1.6632592678070068, + "eval_logits/rejected": 1.235045075416565, + "eval_logps/chosen": -1.0692518949508667, + "eval_logps/rejected": -2.7428486347198486, + "eval_loss": 0.5021397471427917, + "eval_rewards/accuracies": 0.692307710647583, + "eval_rewards/chosen": -1.6038777828216553, + "eval_rewards/margins": 2.510395050048828, + "eval_rewards/rejected": -4.1142730712890625, + "eval_runtime": 25.2582, + "eval_samples_per_second": 28.822, + "eval_steps_per_second": 3.603, + "step": 350 + }, + { + "epoch": 0.3197158081705151, + "grad_norm": 0.8040274381637573, + "learning_rate": 4.322421568553529e-06, + "logits/chosen": 1.187036395072937, + "logits/rejected": 0.4290788769721985, + "logps/chosen": -1.1015206575393677, + "logps/rejected": -2.919748544692993, + "loss": 0.489, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.6522810459136963, + "rewards/margins": 2.727341651916504, + "rewards/rejected": -4.379622459411621, + "step": 360 + }, + { + "epoch": 0.3285968028419183, + "grad_norm": 0.9299562573432922, + "learning_rate": 4.286181699082008e-06, + "logits/chosen": 2.5852127075195312, + "logits/rejected": 2.0419259071350098, + "logps/chosen": -1.1498607397079468, + "logps/rejected": -3.0336194038391113, + "loss": 0.4812, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.724791169166565, + "rewards/margins": 2.8256375789642334, + "rewards/rejected": -4.55042839050293, + "step": 370 + }, + { + "epoch": 0.33747779751332146, + "grad_norm": 1.7739671468734741, + "learning_rate": 4.249158351283414e-06, + "logits/chosen": 2.246245861053467, + "logits/rejected": 1.5551975965499878, + "logps/chosen": -1.254900574684143, + "logps/rejected": -3.206178665161133, + "loss": 0.4651, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -1.8823509216308594, + "rewards/margins": 2.926917552947998, + "rewards/rejected": -4.809267997741699, + "step": 380 + }, + { + "epoch": 0.3463587921847247, + "grad_norm": 4.380665302276611, + "learning_rate": 4.211367764821722e-06, + "logits/chosen": 3.0754549503326416, + "logits/rejected": 2.622124433517456, + "logps/chosen": -1.9250037670135498, + "logps/rejected": -3.69482421875, + "loss": 0.4292, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.8875060081481934, + "rewards/margins": 2.6547305583953857, + "rewards/rejected": -5.542236328125, + "step": 390 + }, + { + "epoch": 0.3552397868561279, + "grad_norm": 1.5087212324142456, + "learning_rate": 4.172826515897146e-06, + "logits/chosen": 2.2718021869659424, + "logits/rejected": 1.8861210346221924, + "logps/chosen": -2.4473955631256104, + "logps/rejected": -4.387387752532959, + "loss": 0.3902, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -3.671093702316284, + "rewards/margins": 2.9099888801574707, + "rewards/rejected": -6.581082344055176, + "step": 400 + }, + { + "epoch": 0.3552397868561279, + "eval_logits/chosen": 1.759078860282898, + "eval_logits/rejected": 1.5246928930282593, + "eval_logps/chosen": -2.720665454864502, + "eval_logps/rejected": -4.613493919372559, + "eval_loss": 0.4054907560348511, + "eval_rewards/accuracies": 0.8791208863258362, + "eval_rewards/chosen": -4.080998420715332, + "eval_rewards/margins": 2.839242696762085, + "eval_rewards/rejected": -6.920241355895996, + "eval_runtime": 25.2363, + "eval_samples_per_second": 28.847, + "eval_steps_per_second": 3.606, + "step": 400 + }, + { + "epoch": 0.3641207815275311, + "grad_norm": 6.079421043395996, + "learning_rate": 4.133551509975264e-06, + "logits/chosen": 1.8841949701309204, + "logits/rejected": 1.3479797840118408, + "logps/chosen": -2.517265796661377, + "logps/rejected": -4.453648567199707, + "loss": 0.3977, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -3.7758986949920654, + "rewards/margins": 2.9045748710632324, + "rewards/rejected": -6.680473327636719, + "step": 410 + }, + { + "epoch": 0.37300177619893427, + "grad_norm": 3.0998194217681885, + "learning_rate": 4.093559974371725e-06, + "logits/chosen": 1.6409276723861694, + "logits/rejected": 1.2141990661621094, + "logps/chosen": -2.2561168670654297, + "logps/rejected": -4.470211029052734, + "loss": 0.3527, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.3841750621795654, + "rewards/margins": 3.321141004562378, + "rewards/rejected": -6.70531702041626, + "step": 420 + }, + { + "epoch": 0.38188277087033745, + "grad_norm": 6.982161045074463, + "learning_rate": 4.052869450695776e-06, + "logits/chosen": 2.835188388824463, + "logits/rejected": 2.3657329082489014, + "logps/chosen": -2.8557300567626953, + "logps/rejected": -5.075521469116211, + "loss": 0.387, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -4.283595085144043, + "rewards/margins": 3.3296875953674316, + "rewards/rejected": -7.613282680511475, + "step": 430 + }, + { + "epoch": 0.3907637655417407, + "grad_norm": 2.139338970184326, + "learning_rate": 4.011497787155938e-06, + "logits/chosen": 2.126509189605713, + "logits/rejected": 1.459567904472351, + "logps/chosen": -3.1412863731384277, + "logps/rejected": -5.423466682434082, + "loss": 0.3611, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -4.711928844451904, + "rewards/margins": 3.4232699871063232, + "rewards/rejected": -8.135198593139648, + "step": 440 + }, + { + "epoch": 0.3996447602131439, + "grad_norm": 1.7899377346038818, + "learning_rate": 3.969463130731183e-06, + "logits/chosen": 2.4551379680633545, + "logits/rejected": 2.0784289836883545, + "logps/chosen": -3.098043203353882, + "logps/rejected": -5.300747871398926, + "loss": 0.354, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -4.647065162658691, + "rewards/margins": 3.3040566444396973, + "rewards/rejected": -7.951122283935547, + "step": 450 + }, + { + "epoch": 0.3996447602131439, + "eval_logits/chosen": 1.9761625528335571, + "eval_logits/rejected": 1.6654667854309082, + "eval_logps/chosen": -2.8789772987365723, + "eval_logps/rejected": -5.1105055809021, + "eval_loss": 0.36211252212524414, + "eval_rewards/accuracies": 0.8791208863258362, + "eval_rewards/chosen": -4.3184661865234375, + "eval_rewards/margins": 3.347292423248291, + "eval_rewards/rejected": -7.6657586097717285, + "eval_runtime": 25.2549, + "eval_samples_per_second": 28.826, + "eval_steps_per_second": 3.603, + "step": 450 + }, + { + "epoch": 0.40852575488454707, + "grad_norm": 1.9171936511993408, + "learning_rate": 3.92678391921108e-06, + "logits/chosen": 2.1672446727752686, + "logits/rejected": 1.6228408813476562, + "logps/chosen": -2.688931703567505, + "logps/rejected": -5.2408246994018555, + "loss": 0.3266, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -4.033397197723389, + "rewards/margins": 3.8278393745422363, + "rewards/rejected": -7.861237525939941, + "step": 460 + }, + { + "epoch": 0.41740674955595025, + "grad_norm": 1.702635407447815, + "learning_rate": 3.88347887310836e-06, + "logits/chosen": 2.3164448738098145, + "logits/rejected": 2.047529697418213, + "logps/chosen": -2.6861701011657715, + "logps/rejected": -5.629918098449707, + "loss": 0.3339, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -4.02925443649292, + "rewards/margins": 4.415622711181641, + "rewards/rejected": -8.444877624511719, + "step": 470 + }, + { + "epoch": 0.42628774422735344, + "grad_norm": 2.48634934425354, + "learning_rate": 3.839566987447492e-06, + "logits/chosen": 2.5225472450256348, + "logits/rejected": 2.0870003700256348, + "logps/chosen": -3.041111946105957, + "logps/rejected": -5.3499016761779785, + "loss": 0.3226, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.561667442321777, + "rewards/margins": 3.4631850719451904, + "rewards/rejected": -8.024852752685547, + "step": 480 + }, + { + "epoch": 0.4351687388987567, + "grad_norm": 4.728499412536621, + "learning_rate": 3.795067523432826e-06, + "logits/chosen": 2.33893084526062, + "logits/rejected": 1.7909936904907227, + "logps/chosen": -2.7356209754943848, + "logps/rejected": -5.33417272567749, + "loss": 0.322, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.103431224822998, + "rewards/margins": 3.8978283405303955, + "rewards/rejected": -8.001258850097656, + "step": 490 + }, + { + "epoch": 0.44404973357015987, + "grad_norm": 8.412679672241211, + "learning_rate": 3.7500000000000005e-06, + "logits/chosen": 2.788668632507324, + "logits/rejected": 2.439873695373535, + "logps/chosen": -3.3219153881073, + "logps/rejected": -5.992051124572754, + "loss": 0.3075, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.98287296295166, + "rewards/margins": 4.005204200744629, + "rewards/rejected": -8.988077163696289, + "step": 500 + }, + { + "epoch": 0.44404973357015987, + "eval_logits/chosen": 2.165436029434204, + "eval_logits/rejected": 1.8186790943145752, + "eval_logps/chosen": -3.4299349784851074, + "eval_logps/rejected": -6.0660552978515625, + "eval_loss": 0.3319137990474701, + "eval_rewards/accuracies": 0.8901098966598511, + "eval_rewards/chosen": -5.14490270614624, + "eval_rewards/margins": 3.954181671142578, + "eval_rewards/rejected": -9.09908390045166, + "eval_runtime": 25.2602, + "eval_samples_per_second": 28.82, + "eval_steps_per_second": 3.603, + "step": 500 + }, + { + "epoch": 0.45293072824156305, + "grad_norm": 2.8339133262634277, + "learning_rate": 3.7043841852542884e-06, + "logits/chosen": 2.109018325805664, + "logits/rejected": 1.6996265649795532, + "logps/chosen": -3.288560390472412, + "logps/rejected": -5.986764430999756, + "loss": 0.3116, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -4.932840824127197, + "rewards/margins": 4.047306060791016, + "rewards/rejected": -8.980146408081055, + "step": 510 + }, + { + "epoch": 0.46181172291296624, + "grad_norm": 3.8578269481658936, + "learning_rate": 3.658240087799655e-06, + "logits/chosen": 1.7659969329833984, + "logits/rejected": 1.4596515893936157, + "logps/chosen": -3.0301425457000732, + "logps/rejected": -6.252682209014893, + "loss": 0.3015, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -4.5452141761779785, + "rewards/margins": 4.833809852600098, + "rewards/rejected": -9.379022598266602, + "step": 520 + }, + { + "epoch": 0.4706927175843694, + "grad_norm": 2.7795143127441406, + "learning_rate": 3.611587947962319e-06, + "logits/chosen": 2.472006320953369, + "logits/rejected": 1.9400993585586548, + "logps/chosen": -3.2479186058044434, + "logps/rejected": -6.475512504577637, + "loss": 0.3006, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -4.871877193450928, + "rewards/margins": 4.841391086578369, + "rewards/rejected": -9.713269233703613, + "step": 530 + }, + { + "epoch": 0.47957371225577267, + "grad_norm": 3.5200746059417725, + "learning_rate": 3.564448228912682e-06, + "logits/chosen": 2.911531925201416, + "logits/rejected": 2.2947440147399902, + "logps/chosen": -3.387556791305542, + "logps/rejected": -6.559035301208496, + "loss": 0.2684, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -5.081335544586182, + "rewards/margins": 4.757218360900879, + "rewards/rejected": -9.838552474975586, + "step": 540 + }, + { + "epoch": 0.48845470692717585, + "grad_norm": 2.368495225906372, + "learning_rate": 3.516841607689501e-06, + "logits/chosen": 1.276886224746704, + "logits/rejected": 1.3811718225479126, + "logps/chosen": -2.9689180850982666, + "logps/rejected": -6.619606971740723, + "loss": 0.3159, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.4533772468566895, + "rewards/margins": 5.4760332107543945, + "rewards/rejected": -9.929410934448242, + "step": 550 + }, + { + "epoch": 0.48845470692717585, + "eval_logits/chosen": 2.2735824584960938, + "eval_logits/rejected": 1.9788992404937744, + "eval_logps/chosen": -3.698131799697876, + "eval_logps/rejected": -6.64966344833374, + "eval_loss": 0.30747923254966736, + "eval_rewards/accuracies": 0.9230769276618958, + "eval_rewards/chosen": -5.5471978187561035, + "eval_rewards/margins": 4.427298545837402, + "eval_rewards/rejected": -9.974496841430664, + "eval_runtime": 25.2322, + "eval_samples_per_second": 28.852, + "eval_steps_per_second": 3.606, + "step": 550 + }, + { + "epoch": 0.49733570159857904, + "grad_norm": 1.901207685470581, + "learning_rate": 3.4687889661302577e-06, + "logits/chosen": 1.9734981060028076, + "logits/rejected": 1.8617655038833618, + "logps/chosen": -3.464953899383545, + "logps/rejected": -6.746106147766113, + "loss": 0.2912, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -5.1974310874938965, + "rewards/margins": 4.921727180480957, + "rewards/rejected": -10.119158744812012, + "step": 560 + }, + { + "epoch": 0.5062166962699822, + "grad_norm": 3.526299238204956, + "learning_rate": 3.4203113817116955e-06, + "logits/chosen": 3.0836069583892822, + "logits/rejected": 2.75875186920166, + "logps/chosen": -3.5981743335723877, + "logps/rejected": -6.405775547027588, + "loss": 0.3284, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -5.397261142730713, + "rewards/margins": 4.211403846740723, + "rewards/rejected": -9.608665466308594, + "step": 570 + }, + { + "epoch": 0.5150976909413855, + "grad_norm": 2.816272497177124, + "learning_rate": 3.3714301183045382e-06, + "logits/chosen": 3.0068678855895996, + "logits/rejected": 2.466287136077881, + "logps/chosen": -3.7227580547332764, + "logps/rejected": -6.9311113357543945, + "loss": 0.2728, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -5.584136962890625, + "rewards/margins": 4.812530040740967, + "rewards/rejected": -10.39666748046875, + "step": 580 + }, + { + "epoch": 0.5239786856127886, + "grad_norm": 2.433389902114868, + "learning_rate": 3.3221666168464584e-06, + "logits/chosen": 2.9992904663085938, + "logits/rejected": 2.678699254989624, + "logps/chosen": -3.540968418121338, + "logps/rejected": -7.228091239929199, + "loss": 0.2568, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.3114519119262695, + "rewards/margins": 5.530684947967529, + "rewards/rejected": -10.842137336730957, + "step": 590 + }, + { + "epoch": 0.5328596802841918, + "grad_norm": 2.7557125091552734, + "learning_rate": 3.272542485937369e-06, + "logits/chosen": 2.557410478591919, + "logits/rejected": 2.331958770751953, + "logps/chosen": -3.9404635429382324, + "logps/rejected": -7.266766548156738, + "loss": 0.2639, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -5.9106950759887695, + "rewards/margins": 4.9894537925720215, + "rewards/rejected": -10.900148391723633, + "step": 600 + }, + { + "epoch": 0.5328596802841918, + "eval_logits/chosen": 2.561415910720825, + "eval_logits/rejected": 2.2971484661102295, + "eval_logps/chosen": -4.015191555023193, + "eval_logps/rejected": -7.222255229949951, + "eval_loss": 0.28853774070739746, + "eval_rewards/accuracies": 0.9340659379959106, + "eval_rewards/chosen": -6.022787570953369, + "eval_rewards/margins": 4.810595989227295, + "eval_rewards/rejected": -10.833383560180664, + "eval_runtime": 25.2577, + "eval_samples_per_second": 28.823, + "eval_steps_per_second": 3.603, + "step": 600 + }, + { + "epoch": 0.5417406749555951, + "grad_norm": 2.582770824432373, + "learning_rate": 3.222579492361179e-06, + "logits/chosen": 2.7404181957244873, + "logits/rejected": 2.540113687515259, + "logps/chosen": -3.9154396057128906, + "logps/rejected": -7.631985664367676, + "loss": 0.2816, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -5.873159408569336, + "rewards/margins": 5.574820041656494, + "rewards/rejected": -11.447979927062988, + "step": 610 + }, + { + "epoch": 0.5506216696269982, + "grad_norm": 3.8167436122894287, + "learning_rate": 3.1722995515381644e-06, + "logits/chosen": 2.445218563079834, + "logits/rejected": 2.288620710372925, + "logps/chosen": -3.7501556873321533, + "logps/rejected": -7.918539524078369, + "loss": 0.2891, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -5.625233173370361, + "rewards/margins": 6.252577304840088, + "rewards/rejected": -11.877809524536133, + "step": 620 + }, + { + "epoch": 0.5595026642984015, + "grad_norm": 3.57536244392395, + "learning_rate": 3.121724717912138e-06, + "logits/chosen": 2.8337388038635254, + "logits/rejected": 2.1241557598114014, + "logps/chosen": -3.7040035724639893, + "logps/rejected": -7.53197717666626, + "loss": 0.2702, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -5.556005477905273, + "rewards/margins": 5.741961479187012, + "rewards/rejected": -11.297966003417969, + "step": 630 + }, + { + "epoch": 0.5683836589698046, + "grad_norm": 3.0520713329315186, + "learning_rate": 3.0708771752766397e-06, + "logits/chosen": 2.5255160331726074, + "logits/rejected": 2.0742428302764893, + "logps/chosen": -3.908573865890503, + "logps/rejected": -7.60653829574585, + "loss": 0.2598, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -5.862860679626465, + "rewards/margins": 5.546946048736572, + "rewards/rejected": -11.409807205200195, + "step": 640 + }, + { + "epoch": 0.5772646536412078, + "grad_norm": 8.067182540893555, + "learning_rate": 3.019779227044398e-06, + "logits/chosen": 1.6965067386627197, + "logits/rejected": 1.644667625427246, + "logps/chosen": -3.6588027477264404, + "logps/rejected": -7.457572937011719, + "loss": 0.2453, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.488204002380371, + "rewards/margins": 5.698155403137207, + "rewards/rejected": -11.186359405517578, + "step": 650 + }, + { + "epoch": 0.5772646536412078, + "eval_logits/chosen": 2.577754020690918, + "eval_logits/rejected": 2.265626907348633, + "eval_logps/chosen": -3.906606435775757, + "eval_logps/rejected": -7.3099446296691895, + "eval_loss": 0.27302286028862, + "eval_rewards/accuracies": 0.9560439586639404, + "eval_rewards/chosen": -5.859910011291504, + "eval_rewards/margins": 5.105007171630859, + "eval_rewards/rejected": -10.964917182922363, + "eval_runtime": 25.2446, + "eval_samples_per_second": 28.838, + "eval_steps_per_second": 3.605, + "step": 650 + }, + { + "epoch": 0.5861456483126111, + "grad_norm": 2.9298255443573, + "learning_rate": 2.9684532864643123e-06, + "logits/chosen": 2.901702404022217, + "logits/rejected": 2.595918655395508, + "logps/chosen": -3.9949543476104736, + "logps/rejected": -7.430356502532959, + "loss": 0.2726, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -5.992431163787842, + "rewards/margins": 5.153104782104492, + "rewards/rejected": -11.145535469055176, + "step": 660 + }, + { + "epoch": 0.5950266429840142, + "grad_norm": 3.085571050643921, + "learning_rate": 2.9169218667902562e-06, + "logits/chosen": 2.8471388816833496, + "logits/rejected": 2.5500330924987793, + "logps/chosen": -4.014147758483887, + "logps/rejected": -7.298794746398926, + "loss": 0.2484, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -6.02122163772583, + "rewards/margins": 4.926970481872559, + "rewards/rejected": -10.94819164276123, + "step": 670 + }, + { + "epoch": 0.6039076376554174, + "grad_norm": 2.3615477085113525, + "learning_rate": 2.8652075714060296e-06, + "logits/chosen": 2.495004177093506, + "logits/rejected": 1.9873936176300049, + "logps/chosen": -4.2625041007995605, + "logps/rejected": -8.186586380004883, + "loss": 0.2144, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -6.393756866455078, + "rewards/margins": 5.886124610900879, + "rewards/rejected": -12.279881477355957, + "step": 680 + }, + { + "epoch": 0.6127886323268206, + "grad_norm": 3.497316837310791, + "learning_rate": 2.813333083910761e-06, + "logits/chosen": 1.540621042251587, + "logits/rejected": 1.2002273797988892, + "logps/chosen": -3.71490740776062, + "logps/rejected": -8.136199951171875, + "loss": 0.2861, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -5.572361469268799, + "rewards/margins": 6.631939888000488, + "rewards/rejected": -12.204300880432129, + "step": 690 + }, + { + "epoch": 0.6216696269982238, + "grad_norm": 3.2540247440338135, + "learning_rate": 2.761321158169134e-06, + "logits/chosen": 2.776721477508545, + "logits/rejected": 2.475888729095459, + "logps/chosen": -4.3385329246521, + "logps/rejected": -7.8828301429748535, + "loss": 0.2885, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -6.507800102233887, + "rewards/margins": 5.316445350646973, + "rewards/rejected": -11.824244499206543, + "step": 700 + }, + { + "epoch": 0.6216696269982238, + "eval_logits/chosen": 2.4267516136169434, + "eval_logits/rejected": 2.1245739459991455, + "eval_logps/chosen": -3.9584598541259766, + "eval_logps/rejected": -7.546706199645996, + "eval_loss": 0.26512712240219116, + "eval_rewards/accuracies": 0.9340659379959106, + "eval_rewards/chosen": -5.937689781188965, + "eval_rewards/margins": 5.382368564605713, + "eval_rewards/rejected": -11.320058822631836, + "eval_runtime": 25.2589, + "eval_samples_per_second": 28.822, + "eval_steps_per_second": 3.603, + "step": 700 + }, + { + "epoch": 0.6305506216696269, + "grad_norm": 2.582273483276367, + "learning_rate": 2.70919460833079e-06, + "logits/chosen": 2.2102553844451904, + "logits/rejected": 1.9504516124725342, + "logps/chosen": -3.7470879554748535, + "logps/rejected": -7.761659145355225, + "loss": 0.2078, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -5.620632171630859, + "rewards/margins": 6.02185583114624, + "rewards/rejected": -11.642488479614258, + "step": 710 + }, + { + "epoch": 0.6394316163410302, + "grad_norm": 2.2855663299560547, + "learning_rate": 2.6569762988232838e-06, + "logits/chosen": 2.4659409523010254, + "logits/rejected": 2.0434811115264893, + "logps/chosen": -3.4346442222595215, + "logps/rejected": -7.470824241638184, + "loss": 0.2346, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.151965618133545, + "rewards/margins": 6.054270267486572, + "rewards/rejected": -11.206236839294434, + "step": 720 + }, + { + "epoch": 0.6483126110124334, + "grad_norm": 2.038733959197998, + "learning_rate": 2.604689134322999e-06, + "logits/chosen": 2.270310878753662, + "logits/rejected": 1.9651508331298828, + "logps/chosen": -3.721379518508911, + "logps/rejected": -7.776650905609131, + "loss": 0.2354, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -5.582068920135498, + "rewards/margins": 6.0829057693481445, + "rewards/rejected": -11.6649751663208, + "step": 730 + }, + { + "epoch": 0.6571936056838366, + "grad_norm": 2.948915481567383, + "learning_rate": 2.5523560497083927e-06, + "logits/chosen": 2.358057737350464, + "logits/rejected": 2.063586711883545, + "logps/chosen": -3.6883864402770996, + "logps/rejected": -7.352984428405762, + "loss": 0.231, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -5.5325798988342285, + "rewards/margins": 5.496896266937256, + "rewards/rejected": -11.0294771194458, + "step": 740 + }, + { + "epoch": 0.6660746003552398, + "grad_norm": 3.661870002746582, + "learning_rate": 2.5e-06, + "logits/chosen": 2.6712796688079834, + "logits/rejected": 2.393817901611328, + "logps/chosen": -4.065141201019287, + "logps/rejected": -7.9232306480407715, + "loss": 0.2332, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.097712516784668, + "rewards/margins": 5.787134170532227, + "rewards/rejected": -11.884846687316895, + "step": 750 + }, + { + "epoch": 0.6660746003552398, + "eval_logits/chosen": 2.4871790409088135, + "eval_logits/rejected": 2.2864174842834473, + "eval_logps/chosen": -4.176412105560303, + "eval_logps/rejected": -8.009041786193848, + "eval_loss": 0.2521994411945343, + "eval_rewards/accuracies": 0.9340659379959106, + "eval_rewards/chosen": -6.264617443084717, + "eval_rewards/margins": 5.7489447593688965, + "eval_rewards/rejected": -12.01356315612793, + "eval_runtime": 25.2624, + "eval_samples_per_second": 28.818, + "eval_steps_per_second": 3.602, + "step": 750 + }, + { + "epoch": 0.6749555950266429, + "grad_norm": 1.992043375968933, + "learning_rate": 2.447643950291608e-06, + "logits/chosen": 1.8225914239883423, + "logits/rejected": 1.842559814453125, + "logps/chosen": -4.158253192901611, + "logps/rejected": -8.65810775756836, + "loss": 0.2104, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -6.237379550933838, + "rewards/margins": 6.749783515930176, + "rewards/rejected": -12.987161636352539, + "step": 760 + }, + { + "epoch": 0.6838365896980462, + "grad_norm": 1.9012709856033325, + "learning_rate": 2.3953108656770018e-06, + "logits/chosen": 2.491041660308838, + "logits/rejected": 1.9490203857421875, + "logps/chosen": -4.141987323760986, + "logps/rejected": -8.790396690368652, + "loss": 0.2236, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -6.212981224060059, + "rewards/margins": 6.9726152420043945, + "rewards/rejected": -13.185595512390137, + "step": 770 + }, + { + "epoch": 0.6927175843694494, + "grad_norm": 3.4377241134643555, + "learning_rate": 2.3430237011767166e-06, + "logits/chosen": 2.54376220703125, + "logits/rejected": 2.2269883155822754, + "logps/chosen": -3.7832961082458496, + "logps/rejected": -8.058969497680664, + "loss": 0.2449, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -5.674944877624512, + "rewards/margins": 6.413510322570801, + "rewards/rejected": -12.08845329284668, + "step": 780 + }, + { + "epoch": 0.7015985790408525, + "grad_norm": 1.365478277206421, + "learning_rate": 2.290805391669212e-06, + "logits/chosen": 3.386944532394409, + "logits/rejected": 2.942134141921997, + "logps/chosen": -3.8051178455352783, + "logps/rejected": -8.201702117919922, + "loss": 0.1764, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.707675933837891, + "rewards/margins": 6.594876289367676, + "rewards/rejected": -12.302552223205566, + "step": 790 + }, + { + "epoch": 0.7104795737122558, + "grad_norm": 3.3714189529418945, + "learning_rate": 2.238678841830867e-06, + "logits/chosen": 2.690781593322754, + "logits/rejected": 2.2246782779693604, + "logps/chosen": -4.251287937164307, + "logps/rejected": -8.268930435180664, + "loss": 0.2429, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.376931667327881, + "rewards/margins": 6.026463031768799, + "rewards/rejected": -12.40339469909668, + "step": 800 + }, + { + "epoch": 0.7104795737122558, + "eval_logits/chosen": 2.453395128250122, + "eval_logits/rejected": 2.2390329837799072, + "eval_logps/chosen": -4.038145065307617, + "eval_logps/rejected": -8.012212753295898, + "eval_loss": 0.2468602955341339, + "eval_rewards/accuracies": 0.9340659379959106, + "eval_rewards/chosen": -6.057217597961426, + "eval_rewards/margins": 5.961101055145264, + "eval_rewards/rejected": -12.018318176269531, + "eval_runtime": 25.2622, + "eval_samples_per_second": 28.818, + "eval_steps_per_second": 3.602, + "step": 800 + }, + { + "epoch": 0.7193605683836589, + "grad_norm": 4.527968883514404, + "learning_rate": 2.186666916089239e-06, + "logits/chosen": 2.250300168991089, + "logits/rejected": 2.254126787185669, + "logps/chosen": -4.0314741134643555, + "logps/rejected": -7.800264835357666, + "loss": 0.2277, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -6.047211647033691, + "rewards/margins": 5.65318489074707, + "rewards/rejected": -11.700395584106445, + "step": 810 + }, + { + "epoch": 0.7282415630550622, + "grad_norm": 3.3826537132263184, + "learning_rate": 2.134792428593971e-06, + "logits/chosen": 2.260606050491333, + "logits/rejected": 1.8174827098846436, + "logps/chosen": -3.6692988872528076, + "logps/rejected": -7.9340972900390625, + "loss": 0.2428, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -5.503948211669922, + "rewards/margins": 6.397197723388672, + "rewards/rejected": -11.901147842407227, + "step": 820 + }, + { + "epoch": 0.7371225577264654, + "grad_norm": 5.055661678314209, + "learning_rate": 2.0830781332097446e-06, + "logits/chosen": 2.4391427040100098, + "logits/rejected": 2.0999953746795654, + "logps/chosen": -3.677706241607666, + "logps/rejected": -7.840268135070801, + "loss": 0.2288, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -5.516559600830078, + "rewards/margins": 6.243842124938965, + "rewards/rejected": -11.760400772094727, + "step": 830 + }, + { + "epoch": 0.7460035523978685, + "grad_norm": 9.756246566772461, + "learning_rate": 2.031546713535688e-06, + "logits/chosen": 2.5405280590057373, + "logits/rejected": 2.3721981048583984, + "logps/chosen": -3.955289840698242, + "logps/rejected": -7.609635353088379, + "loss": 0.2384, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -5.932934761047363, + "rewards/margins": 5.481517314910889, + "rewards/rejected": -11.41445255279541, + "step": 840 + }, + { + "epoch": 0.7548845470692718, + "grad_norm": 3.4563076496124268, + "learning_rate": 1.9802207729556023e-06, + "logits/chosen": 3.1546592712402344, + "logits/rejected": 2.6494877338409424, + "logps/chosen": -3.9850029945373535, + "logps/rejected": -8.197628021240234, + "loss": 0.2229, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -5.977504253387451, + "rewards/margins": 6.3189377784729, + "rewards/rejected": -12.296442031860352, + "step": 850 + }, + { + "epoch": 0.7548845470692718, + "eval_logits/chosen": 2.536773443222046, + "eval_logits/rejected": 2.2915148735046387, + "eval_logps/chosen": -4.12998628616333, + "eval_logps/rejected": -8.214204788208008, + "eval_loss": 0.2361687868833542, + "eval_rewards/accuracies": 0.9340659379959106, + "eval_rewards/chosen": -6.194979190826416, + "eval_rewards/margins": 6.126327991485596, + "eval_rewards/rejected": -12.321306228637695, + "eval_runtime": 25.2618, + "eval_samples_per_second": 28.818, + "eval_steps_per_second": 3.602, + "step": 850 + }, + { + "epoch": 0.7637655417406749, + "grad_norm": 2.1946489810943604, + "learning_rate": 1.9291228247233607e-06, + "logits/chosen": 1.6681854724884033, + "logits/rejected": 1.4272549152374268, + "logps/chosen": -3.6183536052703857, + "logps/rejected": -8.374089241027832, + "loss": 0.2142, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -5.427530288696289, + "rewards/margins": 7.133603572845459, + "rewards/rejected": -12.56113338470459, + "step": 860 + }, + { + "epoch": 0.7726465364120781, + "grad_norm": 3.251861095428467, + "learning_rate": 1.8782752820878636e-06, + "logits/chosen": 3.6365694999694824, + "logits/rejected": 3.002140522003174, + "logps/chosen": -4.218847274780273, + "logps/rejected": -8.535767555236816, + "loss": 0.1927, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -6.328269958496094, + "rewards/margins": 6.475382328033447, + "rewards/rejected": -12.8036527633667, + "step": 870 + }, + { + "epoch": 0.7815275310834814, + "grad_norm": 2.358853816986084, + "learning_rate": 1.827700448461836e-06, + "logits/chosen": 2.647890567779541, + "logits/rejected": 2.1392157077789307, + "logps/chosen": -4.547138214111328, + "logps/rejected": -9.499224662780762, + "loss": 0.1945, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -6.820706844329834, + "rewards/margins": 7.42812967300415, + "rewards/rejected": -14.2488374710083, + "step": 880 + }, + { + "epoch": 0.7904085257548845, + "grad_norm": 2.04150390625, + "learning_rate": 1.7774205076388207e-06, + "logits/chosen": 2.7666563987731934, + "logits/rejected": 2.417945623397827, + "logps/chosen": -4.042238235473633, + "logps/rejected": -7.750130653381348, + "loss": 0.2228, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -6.063356876373291, + "rewards/margins": 5.561838626861572, + "rewards/rejected": -11.625195503234863, + "step": 890 + }, + { + "epoch": 0.7992895204262878, + "grad_norm": 3.5930328369140625, + "learning_rate": 1.7274575140626318e-06, + "logits/chosen": 2.7729713916778564, + "logits/rejected": 2.426771640777588, + "logps/chosen": -4.228175163269043, + "logps/rejected": -8.267873764038086, + "loss": 0.2134, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -6.342263221740723, + "rewards/margins": 6.05954647064209, + "rewards/rejected": -12.401810646057129, + "step": 900 + }, + { + "epoch": 0.7992895204262878, + "eval_logits/chosen": 2.5573580265045166, + "eval_logits/rejected": 2.317056179046631, + "eval_logps/chosen": -4.389712333679199, + "eval_logps/rejected": -8.52020263671875, + "eval_loss": 0.2303350269794464, + "eval_rewards/accuracies": 0.9450549483299255, + "eval_rewards/chosen": -6.584568023681641, + "eval_rewards/margins": 6.195735931396484, + "eval_rewards/rejected": -12.780303001403809, + "eval_runtime": 25.2651, + "eval_samples_per_second": 28.814, + "eval_steps_per_second": 3.602, + "step": 900 + }, + { + "epoch": 0.8081705150976909, + "grad_norm": 3.19136118888855, + "learning_rate": 1.677833383153542e-06, + "logits/chosen": 2.2394022941589355, + "logits/rejected": 2.2709288597106934, + "logps/chosen": -4.338564872741699, + "logps/rejected": -8.621049880981445, + "loss": 0.2029, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.507847785949707, + "rewards/margins": 6.423727512359619, + "rewards/rejected": -12.931573867797852, + "step": 910 + }, + { + "epoch": 0.8170515097690941, + "grad_norm": 2.785442590713501, + "learning_rate": 1.6285698816954626e-06, + "logits/chosen": 2.669541835784912, + "logits/rejected": 2.38188099861145, + "logps/chosen": -4.103513717651367, + "logps/rejected": -8.361233711242676, + "loss": 0.2054, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -6.155270576477051, + "rewards/margins": 6.386577606201172, + "rewards/rejected": -12.541848182678223, + "step": 920 + }, + { + "epoch": 0.8259325044404974, + "grad_norm": 2.618563175201416, + "learning_rate": 1.5796886182883053e-06, + "logits/chosen": 2.5333399772644043, + "logits/rejected": 2.3547558784484863, + "logps/chosen": -3.701815128326416, + "logps/rejected": -8.315168380737305, + "loss": 0.1958, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -5.552721977233887, + "rewards/margins": 6.9200310707092285, + "rewards/rejected": -12.472753524780273, + "step": 930 + }, + { + "epoch": 0.8348134991119005, + "grad_norm": 2.640681266784668, + "learning_rate": 1.5312110338697427e-06, + "logits/chosen": 2.6260218620300293, + "logits/rejected": 2.625610589981079, + "logps/chosen": -3.761087417602539, + "logps/rejected": -7.872547149658203, + "loss": 0.2147, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -5.64163064956665, + "rewards/margins": 6.167189598083496, + "rewards/rejected": -11.808821678161621, + "step": 940 + }, + { + "epoch": 0.8436944937833037, + "grad_norm": 2.4091708660125732, + "learning_rate": 1.4831583923105e-06, + "logits/chosen": 2.63578462600708, + "logits/rejected": 2.2209556102752686, + "logps/chosen": -4.115639686584473, + "logps/rejected": -8.535353660583496, + "loss": 0.1832, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -6.173459529876709, + "rewards/margins": 6.629571437835693, + "rewards/rejected": -12.803030014038086, + "step": 950 + }, + { + "epoch": 0.8436944937833037, + "eval_logits/chosen": 2.6680469512939453, + "eval_logits/rejected": 2.4164178371429443, + "eval_logps/chosen": -4.110969066619873, + "eval_logps/rejected": -8.377038955688477, + "eval_loss": 0.22471140325069427, + "eval_rewards/accuracies": 0.9560439586639404, + "eval_rewards/chosen": -6.166452884674072, + "eval_rewards/margins": 6.399104118347168, + "eval_rewards/rejected": -12.565558433532715, + "eval_runtime": 25.2357, + "eval_samples_per_second": 28.848, + "eval_steps_per_second": 3.606, + "step": 950 + }, + { + "epoch": 0.8525754884547069, + "grad_norm": 5.924206733703613, + "learning_rate": 1.4355517710873184e-06, + "logits/chosen": 1.8501287698745728, + "logits/rejected": 1.9165807962417603, + "logps/chosen": -3.9117112159729004, + "logps/rejected": -8.62653923034668, + "loss": 0.2384, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -5.86756706237793, + "rewards/margins": 7.072243690490723, + "rewards/rejected": -12.939809799194336, + "step": 960 + }, + { + "epoch": 0.8614564831261101, + "grad_norm": 9.17930793762207, + "learning_rate": 1.388412052037682e-06, + "logits/chosen": 3.1926915645599365, + "logits/rejected": 2.8990261554718018, + "logps/chosen": -3.6797282695770264, + "logps/rejected": -8.219030380249023, + "loss": 0.2089, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -5.51959228515625, + "rewards/margins": 6.808953285217285, + "rewards/rejected": -12.328545570373535, + "step": 970 + }, + { + "epoch": 0.8703374777975134, + "grad_norm": 2.3526487350463867, + "learning_rate": 1.3417599122003464e-06, + "logits/chosen": 3.1998744010925293, + "logits/rejected": 2.904794692993164, + "logps/chosen": -4.254674911499023, + "logps/rejected": -8.549466133117676, + "loss": 0.1936, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.382012367248535, + "rewards/margins": 6.4421868324279785, + "rewards/rejected": -12.824198722839355, + "step": 980 + }, + { + "epoch": 0.8792184724689165, + "grad_norm": 2.372730016708374, + "learning_rate": 1.2956158147457116e-06, + "logits/chosen": 2.3626601696014404, + "logits/rejected": 2.0850844383239746, + "logps/chosen": -4.239551544189453, + "logps/rejected": -8.695666313171387, + "loss": 0.1902, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -6.359326362609863, + "rewards/margins": 6.684172630310059, + "rewards/rejected": -13.043498039245605, + "step": 990 + }, + { + "epoch": 0.8880994671403197, + "grad_norm": 3.703148365020752, + "learning_rate": 1.2500000000000007e-06, + "logits/chosen": 2.5446887016296387, + "logits/rejected": 2.2514405250549316, + "logps/chosen": -4.357522010803223, + "logps/rejected": -8.732219696044922, + "loss": 0.1807, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -6.53628396987915, + "rewards/margins": 6.562047004699707, + "rewards/rejected": -13.0983304977417, + "step": 1000 + }, + { + "epoch": 0.8880994671403197, + "eval_logits/chosen": 2.6361231803894043, + "eval_logits/rejected": 2.4129557609558105, + "eval_logps/chosen": -4.334481239318848, + "eval_logps/rejected": -8.665815353393555, + "eval_loss": 0.22103577852249146, + "eval_rewards/accuracies": 0.9560439586639404, + "eval_rewards/chosen": -6.501720905303955, + "eval_rewards/margins": 6.497002601623535, + "eval_rewards/rejected": -12.998723030090332, + "eval_runtime": 25.2662, + "eval_samples_per_second": 28.813, + "eval_steps_per_second": 3.602, + "step": 1000 + }, + { + "epoch": 0.8969804618117229, + "grad_norm": 3.239999532699585, + "learning_rate": 1.204932476567175e-06, + "logits/chosen": 2.8413162231445312, + "logits/rejected": 2.5842719078063965, + "logps/chosen": -3.9835076332092285, + "logps/rejected": -8.900358200073242, + "loss": 0.2276, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -5.975261688232422, + "rewards/margins": 7.375277042388916, + "rewards/rejected": -13.35053825378418, + "step": 1010 + }, + { + "epoch": 0.9058614564831261, + "grad_norm": 3.8074100017547607, + "learning_rate": 1.160433012552508e-06, + "logits/chosen": 1.8096859455108643, + "logits/rejected": 1.5240850448608398, + "logps/chosen": -4.373281002044678, + "logps/rejected": -8.784773826599121, + "loss": 0.1907, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.559922218322754, + "rewards/margins": 6.6172380447387695, + "rewards/rejected": -13.177160263061523, + "step": 1020 + }, + { + "epoch": 0.9147424511545293, + "grad_norm": 3.3684775829315186, + "learning_rate": 1.11652112689164e-06, + "logits/chosen": 2.5850119590759277, + "logits/rejected": 2.503263235092163, + "logps/chosen": -4.1612935066223145, + "logps/rejected": -9.02672004699707, + "loss": 0.2483, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -6.241940498352051, + "rewards/margins": 7.298140525817871, + "rewards/rejected": -13.540081977844238, + "step": 1030 + }, + { + "epoch": 0.9236234458259325, + "grad_norm": 4.323958873748779, + "learning_rate": 1.073216080788921e-06, + "logits/chosen": 3.1654298305511475, + "logits/rejected": 2.52232027053833, + "logps/chosen": -4.379122734069824, + "logps/rejected": -8.86447525024414, + "loss": 0.1995, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -6.5686845779418945, + "rewards/margins": 6.728028774261475, + "rewards/rejected": -13.296712875366211, + "step": 1040 + }, + { + "epoch": 0.9325044404973357, + "grad_norm": 2.923391342163086, + "learning_rate": 1.0305368692688175e-06, + "logits/chosen": 3.426027774810791, + "logits/rejected": 2.8201966285705566, + "logps/chosen": -4.2101850509643555, + "logps/rejected": -8.720430374145508, + "loss": 0.2275, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.315278053283691, + "rewards/margins": 6.765368461608887, + "rewards/rejected": -13.080645561218262, + "step": 1050 + }, + { + "epoch": 0.9325044404973357, + "eval_logits/chosen": 2.7177271842956543, + "eval_logits/rejected": 2.45237135887146, + "eval_logps/chosen": -4.310559272766113, + "eval_logps/rejected": -8.67324447631836, + "eval_loss": 0.2168363630771637, + "eval_rewards/accuracies": 0.9670329689979553, + "eval_rewards/chosen": -6.465838432312012, + "eval_rewards/margins": 6.544029235839844, + "eval_rewards/rejected": -13.009867668151855, + "eval_runtime": 25.2426, + "eval_samples_per_second": 28.84, + "eval_steps_per_second": 3.605, + "step": 1050 + }, + { + "epoch": 0.9413854351687388, + "grad_norm": 2.5395636558532715, + "learning_rate": 9.88502212844063e-07, + "logits/chosen": 2.634228467941284, + "logits/rejected": 2.447514772415161, + "logps/chosen": -4.163097381591797, + "logps/rejected": -9.294473648071289, + "loss": 0.214, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -6.244646072387695, + "rewards/margins": 7.697064399719238, + "rewards/rejected": -13.9417085647583, + "step": 1060 + }, + { + "epoch": 0.9502664298401421, + "grad_norm": 2.4016919136047363, + "learning_rate": 9.471305493042243e-07, + "logits/chosen": 3.249488115310669, + "logits/rejected": 3.138061285018921, + "logps/chosen": -4.317265510559082, + "logps/rejected": -9.449271202087402, + "loss": 0.2135, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -6.475898742675781, + "rewards/margins": 7.6980085372924805, + "rewards/rejected": -14.173907279968262, + "step": 1070 + }, + { + "epoch": 0.9591474245115453, + "grad_norm": 7.170376300811768, + "learning_rate": 9.064400256282757e-07, + "logits/chosen": 2.5836658477783203, + "logits/rejected": 2.584625482559204, + "logps/chosen": -3.731645107269287, + "logps/rejected": -8.913705825805664, + "loss": 0.2002, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -5.597468376159668, + "rewards/margins": 7.7730913162231445, + "rewards/rejected": -13.370559692382812, + "step": 1080 + }, + { + "epoch": 0.9680284191829485, + "grad_norm": 4.712115287780762, + "learning_rate": 8.664484900247363e-07, + "logits/chosen": 1.7256263494491577, + "logits/rejected": 1.679078459739685, + "logps/chosen": -4.077349662780762, + "logps/rejected": -9.539873123168945, + "loss": 0.2498, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -6.116024494171143, + "rewards/margins": 8.193784713745117, + "rewards/rejected": -14.309808731079102, + "step": 1090 + }, + { + "epoch": 0.9769094138543517, + "grad_norm": 4.168649673461914, + "learning_rate": 8.271734841028553e-07, + "logits/chosen": 3.224684953689575, + "logits/rejected": 2.831458568572998, + "logps/chosen": -4.505261421203613, + "logps/rejected": -9.360811233520508, + "loss": 0.1854, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.757891654968262, + "rewards/margins": 7.283324241638184, + "rewards/rejected": -14.041216850280762, + "step": 1100 + }, + { + "epoch": 0.9769094138543517, + "eval_logits/chosen": 2.7460875511169434, + "eval_logits/rejected": 2.4983255863189697, + "eval_logps/chosen": -4.352819919586182, + "eval_logps/rejected": -8.75997257232666, + "eval_loss": 0.2113826721906662, + "eval_rewards/accuracies": 0.9780219793319702, + "eval_rewards/chosen": -6.529229640960693, + "eval_rewards/margins": 6.610728740692139, + "eval_rewards/rejected": -13.139957427978516, + "eval_runtime": 25.2523, + "eval_samples_per_second": 28.829, + "eval_steps_per_second": 3.604, + "step": 1100 + }, + { + "epoch": 0.9857904085257548, + "grad_norm": 2.9944748878479004, + "learning_rate": 7.886322351782782e-07, + "logits/chosen": 2.92256236076355, + "logits/rejected": 2.4897289276123047, + "logps/chosen": -4.083096027374268, + "logps/rejected": -8.914036750793457, + "loss": 0.1986, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -6.124643325805664, + "rewards/margins": 7.2464118003845215, + "rewards/rejected": -13.371055603027344, + "step": 1110 + }, + { + "epoch": 0.9946714031971581, + "grad_norm": 3.546858310699463, + "learning_rate": 7.508416487165862e-07, + "logits/chosen": 2.6310508251190186, + "logits/rejected": 2.64538836479187, + "logps/chosen": -4.345538139343262, + "logps/rejected": -9.072227478027344, + "loss": 0.1663, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.518306732177734, + "rewards/margins": 7.090034484863281, + "rewards/rejected": -13.6083402633667, + "step": 1120 + }, + { + "epoch": 1.0035523978685612, + "grad_norm": 2.745328903198242, + "learning_rate": 7.138183009179922e-07, + "logits/chosen": 2.1050074100494385, + "logits/rejected": 2.0917954444885254, + "logps/chosen": -3.694542646408081, + "logps/rejected": -8.503544807434082, + "loss": 0.1985, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -5.541813850402832, + "rewards/margins": 7.213502407073975, + "rewards/rejected": -12.755315780639648, + "step": 1130 + }, + { + "epoch": 1.0124333925399644, + "grad_norm": 1.995431661605835, + "learning_rate": 6.775784314464717e-07, + "logits/chosen": 2.7968056201934814, + "logits/rejected": 2.488126754760742, + "logps/chosen": -4.206582546234131, + "logps/rejected": -8.700386047363281, + "loss": 0.2141, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -6.309874534606934, + "rewards/margins": 6.740703582763672, + "rewards/rejected": -13.050578117370605, + "step": 1140 + }, + { + "epoch": 1.0213143872113677, + "grad_norm": 3.110943078994751, + "learning_rate": 6.421379363065142e-07, + "logits/chosen": 2.5061817169189453, + "logits/rejected": 2.342125415802002, + "logps/chosen": -4.2504048347473145, + "logps/rejected": -8.307449340820312, + "loss": 0.2069, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.375607013702393, + "rewards/margins": 6.085566520690918, + "rewards/rejected": -12.461172103881836, + "step": 1150 + }, + { + "epoch": 1.0213143872113677, + "eval_logits/chosen": 2.728573799133301, + "eval_logits/rejected": 2.5072758197784424, + "eval_logps/chosen": -4.397165298461914, + "eval_logps/rejected": -8.84023380279541, + "eval_loss": 0.20922726392745972, + "eval_rewards/accuracies": 0.9780219793319702, + "eval_rewards/chosen": -6.595747947692871, + "eval_rewards/margins": 6.664604663848877, + "eval_rewards/rejected": -13.26035213470459, + "eval_runtime": 25.264, + "eval_samples_per_second": 28.816, + "eval_steps_per_second": 3.602, + "step": 1150 + }, + { + "epoch": 1.030195381882771, + "grad_norm": 3.401503086090088, + "learning_rate": 6.075123608706093e-07, + "logits/chosen": 2.933258533477783, + "logits/rejected": 2.5766117572784424, + "logps/chosen": -4.0520782470703125, + "logps/rejected": -9.317255973815918, + "loss": 0.2031, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.078117847442627, + "rewards/margins": 7.897767066955566, + "rewards/rejected": -13.975885391235352, + "step": 1160 + }, + { + "epoch": 1.0390763765541742, + "grad_norm": 2.563494920730591, + "learning_rate": 5.737168930605272e-07, + "logits/chosen": 2.8566126823425293, + "logits/rejected": 2.501013994216919, + "logps/chosen": -4.048529148101807, + "logps/rejected": -8.981905937194824, + "loss": 0.2078, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.072793483734131, + "rewards/margins": 7.400063991546631, + "rewards/rejected": -13.472857475280762, + "step": 1170 + }, + { + "epoch": 1.0479573712255772, + "grad_norm": 8.221929550170898, + "learning_rate": 5.407663566854008e-07, + "logits/chosen": 1.8076190948486328, + "logits/rejected": 1.5266883373260498, + "logps/chosen": -3.697718858718872, + "logps/rejected": -8.30528450012207, + "loss": 0.1869, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -5.546578407287598, + "rewards/margins": 6.911346435546875, + "rewards/rejected": -12.457925796508789, + "step": 1180 + }, + { + "epoch": 1.0568383658969804, + "grad_norm": 6.9954400062561035, + "learning_rate": 5.086752049395094e-07, + "logits/chosen": 2.3529181480407715, + "logits/rejected": 2.250333786010742, + "logps/chosen": -4.018799781799316, + "logps/rejected": -8.714046478271484, + "loss": 0.2026, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.028199195861816, + "rewards/margins": 7.042870998382568, + "rewards/rejected": -13.071070671081543, + "step": 1190 + }, + { + "epoch": 1.0657193605683837, + "grad_norm": 3.6167593002319336, + "learning_rate": 4.774575140626317e-07, + "logits/chosen": 2.7552218437194824, + "logits/rejected": 2.428560733795166, + "logps/chosen": -4.324695587158203, + "logps/rejected": -9.764104843139648, + "loss": 0.2233, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.487043857574463, + "rewards/margins": 8.159114837646484, + "rewards/rejected": -14.646158218383789, + "step": 1200 + }, + { + "epoch": 1.0657193605683837, + "eval_logits/chosen": 2.7005860805511475, + "eval_logits/rejected": 2.4944183826446533, + "eval_logps/chosen": -4.283793926239014, + "eval_logps/rejected": -8.801348686218262, + "eval_loss": 0.20932108163833618, + "eval_rewards/accuracies": 0.9560439586639404, + "eval_rewards/chosen": -6.425690650939941, + "eval_rewards/margins": 6.776332855224609, + "eval_rewards/rejected": -13.202025413513184, + "eval_runtime": 25.2698, + "eval_samples_per_second": 28.809, + "eval_steps_per_second": 3.601, + "step": 1200 + }, + { + "epoch": 1.074600355239787, + "grad_norm": 3.2438127994537354, + "learning_rate": 4.4712697716573994e-07, + "logits/chosen": 1.9486421346664429, + "logits/rejected": 1.7296040058135986, + "logps/chosen": -3.9212806224823, + "logps/rejected": -8.304320335388184, + "loss": 0.1761, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -5.881921291351318, + "rewards/margins": 6.574559688568115, + "rewards/rejected": -12.45648193359375, + "step": 1210 + }, + { + "epoch": 1.0834813499111902, + "grad_norm": 2.984710454940796, + "learning_rate": 4.1769689822475147e-07, + "logits/chosen": 3.56829833984375, + "logits/rejected": 2.9676673412323, + "logps/chosen": -4.453071117401123, + "logps/rejected": -8.549017906188965, + "loss": 0.1915, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.6796064376831055, + "rewards/margins": 6.1439208984375, + "rewards/rejected": -12.823527336120605, + "step": 1220 + }, + { + "epoch": 1.0923623445825932, + "grad_norm": 1.0289857387542725, + "learning_rate": 3.891801862449629e-07, + "logits/chosen": 2.529848098754883, + "logits/rejected": 2.3582332134246826, + "logps/chosen": -3.964491367340088, + "logps/rejected": -9.1004638671875, + "loss": 0.1743, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -5.946737289428711, + "rewards/margins": 7.703960418701172, + "rewards/rejected": -13.650697708129883, + "step": 1230 + }, + { + "epoch": 1.1012433392539964, + "grad_norm": 3.8722472190856934, + "learning_rate": 3.615893495987335e-07, + "logits/chosen": 2.980180263519287, + "logits/rejected": 2.823392152786255, + "logps/chosen": -4.560516357421875, + "logps/rejected": -9.179193496704102, + "loss": 0.195, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -6.8407745361328125, + "rewards/margins": 6.928016662597656, + "rewards/rejected": -13.768791198730469, + "step": 1240 + }, + { + "epoch": 1.1101243339253997, + "grad_norm": 3.892000198364258, + "learning_rate": 3.3493649053890325e-07, + "logits/chosen": 2.8612444400787354, + "logits/rejected": 2.5222373008728027, + "logps/chosen": -4.044814586639404, + "logps/rejected": -8.645524978637695, + "loss": 0.1571, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.067221641540527, + "rewards/margins": 6.901064872741699, + "rewards/rejected": -12.968287467956543, + "step": 1250 + }, + { + "epoch": 1.1101243339253997, + "eval_logits/chosen": 2.714733839035034, + "eval_logits/rejected": 2.5066633224487305, + "eval_logps/chosen": -4.303951263427734, + "eval_logps/rejected": -8.823600769042969, + "eval_loss": 0.2080618292093277, + "eval_rewards/accuracies": 0.9560439586639404, + "eval_rewards/chosen": -6.45592737197876, + "eval_rewards/margins": 6.77947473526001, + "eval_rewards/rejected": -13.235400199890137, + "eval_runtime": 25.2645, + "eval_samples_per_second": 28.815, + "eval_steps_per_second": 3.602, + "step": 1250 + }, + { + "epoch": 1.119005328596803, + "grad_norm": 3.225729465484619, + "learning_rate": 3.092332998903416e-07, + "logits/chosen": 2.969749927520752, + "logits/rejected": 2.601804733276367, + "logps/chosen": -4.048078536987305, + "logps/rejected": -8.416013717651367, + "loss": 0.2137, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": -6.072117805480957, + "rewards/margins": 6.55190372467041, + "rewards/rejected": -12.62402057647705, + "step": 1260 + }, + { + "epoch": 1.1278863232682061, + "grad_norm": 3.2009847164154053, + "learning_rate": 2.844910519219632e-07, + "logits/chosen": 2.962148904800415, + "logits/rejected": 2.33172607421875, + "logps/chosen": -4.167851448059082, + "logps/rejected": -8.694719314575195, + "loss": 0.2259, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.2517781257629395, + "rewards/margins": 6.790299892425537, + "rewards/rejected": -13.042078971862793, + "step": 1270 + }, + { + "epoch": 1.1367673179396092, + "grad_norm": 11.98656940460205, + "learning_rate": 2.6072059940146775e-07, + "logits/chosen": 2.3897571563720703, + "logits/rejected": 2.1554014682769775, + "logps/chosen": -4.117510795593262, + "logps/rejected": -9.516136169433594, + "loss": 0.1657, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -6.176266193389893, + "rewards/margins": 8.097938537597656, + "rewards/rejected": -14.274205207824707, + "step": 1280 + }, + { + "epoch": 1.1456483126110124, + "grad_norm": 2.14774489402771, + "learning_rate": 2.3793236883495164e-07, + "logits/chosen": 2.2359044551849365, + "logits/rejected": 2.196018934249878, + "logps/chosen": -4.038699150085449, + "logps/rejected": -8.689764022827148, + "loss": 0.1965, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.058049201965332, + "rewards/margins": 6.976596832275391, + "rewards/rejected": -13.034646987915039, + "step": 1290 + }, + { + "epoch": 1.1545293072824157, + "grad_norm": 3.720881462097168, + "learning_rate": 2.1613635589349756e-07, + "logits/chosen": 2.0333285331726074, + "logits/rejected": 1.8229951858520508, + "logps/chosen": -3.95289945602417, + "logps/rejected": -9.260812759399414, + "loss": 0.174, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -5.929348945617676, + "rewards/margins": 7.9618706703186035, + "rewards/rejected": -13.891218185424805, + "step": 1300 + }, + { + "epoch": 1.1545293072824157, + "eval_logits/chosen": 2.7231945991516113, + "eval_logits/rejected": 2.5083000659942627, + "eval_logps/chosen": -4.294267177581787, + "eval_logps/rejected": -8.827666282653809, + "eval_loss": 0.20604319870471954, + "eval_rewards/accuracies": 0.9670329689979553, + "eval_rewards/chosen": -6.44140100479126, + "eval_rewards/margins": 6.800098419189453, + "eval_rewards/rejected": -13.241500854492188, + "eval_runtime": 25.2569, + "eval_samples_per_second": 28.824, + "eval_steps_per_second": 3.603, + "step": 1300 + }, + { + "epoch": 1.163410301953819, + "grad_norm": 5.726924896240234, + "learning_rate": 1.95342121028749e-07, + "logits/chosen": 3.07940936088562, + "logits/rejected": 2.9031105041503906, + "logps/chosen": -4.019009590148926, + "logps/rejected": -9.042330741882324, + "loss": 0.1857, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -6.028514385223389, + "rewards/margins": 7.534982204437256, + "rewards/rejected": -13.563494682312012, + "step": 1310 + }, + { + "epoch": 1.1722912966252221, + "grad_norm": 4.062259197235107, + "learning_rate": 1.7555878527937164e-07, + "logits/chosen": 1.7906630039215088, + "logits/rejected": 1.741253137588501, + "logps/chosen": -3.8779044151306152, + "logps/rejected": -9.477428436279297, + "loss": 0.1816, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -5.816856384277344, + "rewards/margins": 8.399286270141602, + "rewards/rejected": -14.216143608093262, + "step": 1320 + }, + { + "epoch": 1.1811722912966252, + "grad_norm": 4.922858715057373, + "learning_rate": 1.567950262702714e-07, + "logits/chosen": 2.7542059421539307, + "logits/rejected": 2.369173288345337, + "logps/chosen": -4.438471794128418, + "logps/rejected": -9.667543411254883, + "loss": 0.1965, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -6.657707214355469, + "rewards/margins": 7.8436079025268555, + "rewards/rejected": -14.501315116882324, + "step": 1330 + }, + { + "epoch": 1.1900532859680284, + "grad_norm": 3.347116708755493, + "learning_rate": 1.3905907440629752e-07, + "logits/chosen": 2.6956562995910645, + "logits/rejected": 2.3373332023620605, + "logps/chosen": -3.7666268348693848, + "logps/rejected": -9.006341934204102, + "loss": 0.182, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -5.64993953704834, + "rewards/margins": 7.859574317932129, + "rewards/rejected": -13.509511947631836, + "step": 1340 + }, + { + "epoch": 1.1989342806394316, + "grad_norm": 3.2019412517547607, + "learning_rate": 1.223587092621162e-07, + "logits/chosen": 2.480767011642456, + "logits/rejected": 2.2258126735687256, + "logps/chosen": -4.096011161804199, + "logps/rejected": -8.5545072555542, + "loss": 0.2274, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.144017219543457, + "rewards/margins": 6.687742710113525, + "rewards/rejected": -12.831761360168457, + "step": 1350 + }, + { + "epoch": 1.1989342806394316, + "eval_logits/chosen": 2.7050983905792236, + "eval_logits/rejected": 2.495755434036255, + "eval_logps/chosen": -4.344603061676025, + "eval_logps/rejected": -8.87659740447998, + "eval_loss": 0.20770707726478577, + "eval_rewards/accuracies": 0.9780219793319702, + "eval_rewards/chosen": -6.516905307769775, + "eval_rewards/margins": 6.7979912757873535, + "eval_rewards/rejected": -13.314897537231445, + "eval_runtime": 25.266, + "eval_samples_per_second": 28.813, + "eval_steps_per_second": 3.602, + "step": 1350 + }, + { + "epoch": 1.2078152753108349, + "grad_norm": 6.736517906188965, + "learning_rate": 1.067012561698319e-07, + "logits/chosen": 2.2384419441223145, + "logits/rejected": 2.2167811393737793, + "logps/chosen": -3.9598464965820312, + "logps/rejected": -8.5329008102417, + "loss": 0.1977, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.939770221710205, + "rewards/margins": 6.85958194732666, + "rewards/rejected": -12.799351692199707, + "step": 1360 + }, + { + "epoch": 1.2166962699822381, + "grad_norm": 2.889807939529419, + "learning_rate": 9.209358300585474e-08, + "logits/chosen": 1.8230903148651123, + "logits/rejected": 1.7674376964569092, + "logps/chosen": -3.5624186992645264, + "logps/rejected": -8.8640775680542, + "loss": 0.1655, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -5.343628406524658, + "rewards/margins": 7.952487945556641, + "rewards/rejected": -13.296114921569824, + "step": 1370 + }, + { + "epoch": 1.2255772646536411, + "grad_norm": 2.0800833702087402, + "learning_rate": 7.854209717842231e-08, + "logits/chosen": 2.958648204803467, + "logits/rejected": 2.6478304862976074, + "logps/chosen": -4.070685863494873, + "logps/rejected": -8.65223217010498, + "loss": 0.1918, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -6.1060285568237305, + "rewards/margins": 6.872319221496582, + "rewards/rejected": -12.978347778320312, + "step": 1380 + }, + { + "epoch": 1.2344582593250444, + "grad_norm": 4.166500568389893, + "learning_rate": 6.605274281709929e-08, + "logits/chosen": 2.3533079624176025, + "logits/rejected": 2.153561592102051, + "logps/chosen": -3.9434361457824707, + "logps/rejected": -9.156129837036133, + "loss": 0.219, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -5.915154457092285, + "rewards/margins": 7.819040775299072, + "rewards/rejected": -13.7341947555542, + "step": 1390 + }, + { + "epoch": 1.2433392539964476, + "grad_norm": 3.4721784591674805, + "learning_rate": 5.463099816548578e-08, + "logits/chosen": 3.012937068939209, + "logits/rejected": 2.7736458778381348, + "logps/chosen": -4.2211012840271, + "logps/rejected": -9.378921508789062, + "loss": 0.1719, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -6.3316521644592285, + "rewards/margins": 7.736731052398682, + "rewards/rejected": -14.068384170532227, + "step": 1400 + }, + { + "epoch": 1.2433392539964476, + "eval_logits/chosen": 2.7338051795959473, + "eval_logits/rejected": 2.516977310180664, + "eval_logps/chosen": -4.3559651374816895, + "eval_logps/rejected": -8.89287281036377, + "eval_loss": 0.20297038555145264, + "eval_rewards/accuracies": 0.9780219793319702, + "eval_rewards/chosen": -6.533947467803955, + "eval_rewards/margins": 6.805361747741699, + "eval_rewards/rejected": -13.339309692382812, + "eval_runtime": 25.2634, + "eval_samples_per_second": 28.816, + "eval_steps_per_second": 3.602, + "step": 1400 + }, + { + "epoch": 1.2522202486678509, + "grad_norm": 5.4153571128845215, + "learning_rate": 4.428187317827848e-08, + "logits/chosen": 2.693077325820923, + "logits/rejected": 2.754072666168213, + "logps/chosen": -4.117603302001953, + "logps/rejected": -8.823477745056152, + "loss": 0.199, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.1764044761657715, + "rewards/margins": 7.058813571929932, + "rewards/rejected": -13.235217094421387, + "step": 1410 + }, + { + "epoch": 1.261101243339254, + "grad_norm": 5.981139183044434, + "learning_rate": 3.5009907323737826e-08, + "logits/chosen": 2.250978708267212, + "logits/rejected": 1.818576455116272, + "logps/chosen": -4.223572254180908, + "logps/rejected": -9.683964729309082, + "loss": 0.2149, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.335358142852783, + "rewards/margins": 8.19058895111084, + "rewards/rejected": -14.525945663452148, + "step": 1420 + }, + { + "epoch": 1.2699822380106571, + "grad_norm": 2.202054738998413, + "learning_rate": 2.681916759252917e-08, + "logits/chosen": 2.31886625289917, + "logits/rejected": 2.116328716278076, + "logps/chosen": -4.074057579040527, + "logps/rejected": -8.692493438720703, + "loss": 0.1547, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -6.111086368560791, + "rewards/margins": 6.9276533126831055, + "rewards/rejected": -13.038739204406738, + "step": 1430 + }, + { + "epoch": 1.2788632326820604, + "grad_norm": 2.1010451316833496, + "learning_rate": 1.9713246713805588e-08, + "logits/chosen": 2.656672477722168, + "logits/rejected": 2.773162364959717, + "logps/chosen": -4.189755439758301, + "logps/rejected": -9.755474090576172, + "loss": 0.2032, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -6.284633159637451, + "rewards/margins": 8.348578453063965, + "rewards/rejected": -14.633212089538574, + "step": 1440 + }, + { + "epoch": 1.2877442273534636, + "grad_norm": 5.367340564727783, + "learning_rate": 1.3695261579316776e-08, + "logits/chosen": 2.364032030105591, + "logits/rejected": 2.221125602722168, + "logps/chosen": -4.26717472076416, + "logps/rejected": -9.568387985229492, + "loss": 0.1819, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -6.40076208114624, + "rewards/margins": 7.951818943023682, + "rewards/rejected": -14.352581977844238, + "step": 1450 + }, + { + "epoch": 1.2877442273534636, + "eval_logits/chosen": 2.730602502822876, + "eval_logits/rejected": 2.5076637268066406, + "eval_logps/chosen": -4.366745948791504, + "eval_logps/rejected": -8.909856796264648, + "eval_loss": 0.20565223693847656, + "eval_rewards/accuracies": 0.9670329689979553, + "eval_rewards/chosen": -6.550118446350098, + "eval_rewards/margins": 6.814664840698242, + "eval_rewards/rejected": -13.36478328704834, + "eval_runtime": 25.2605, + "eval_samples_per_second": 28.82, + "eval_steps_per_second": 3.602, + "step": 1450 + }, + { + "epoch": 1.2966252220248669, + "grad_norm": 1.8518935441970825, + "learning_rate": 8.767851876239075e-09, + "logits/chosen": 2.369659900665283, + "logits/rejected": 2.388401746749878, + "logps/chosen": -3.962648391723633, + "logps/rejected": -9.064709663391113, + "loss": 0.1783, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -5.943972110748291, + "rewards/margins": 7.653090476989746, + "rewards/rejected": -13.597063064575195, + "step": 1460 + }, + { + "epoch": 1.30550621669627, + "grad_norm": 3.998896360397339, + "learning_rate": 4.933178929321103e-09, + "logits/chosen": 2.891472339630127, + "logits/rejected": 2.6681532859802246, + "logps/chosen": -4.776543140411377, + "logps/rejected": -9.590726852416992, + "loss": 0.1781, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -7.1648149490356445, + "rewards/margins": 7.22127628326416, + "rewards/rejected": -14.386091232299805, + "step": 1470 + }, + { + "epoch": 1.3143872113676731, + "grad_norm": 3.8183224201202393, + "learning_rate": 2.192924752854042e-09, + "logits/chosen": 2.9981322288513184, + "logits/rejected": 2.6835460662841797, + "logps/chosen": -3.977015972137451, + "logps/rejected": -8.659049987792969, + "loss": 0.1521, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -5.965523719787598, + "rewards/margins": 7.023049831390381, + "rewards/rejected": -12.988574028015137, + "step": 1480 + }, + { + "epoch": 1.3232682060390764, + "grad_norm": 4.379500865936279, + "learning_rate": 5.48291312886251e-10, + "logits/chosen": 1.6689484119415283, + "logits/rejected": 1.8210983276367188, + "logps/chosen": -4.481314659118652, + "logps/rejected": -9.47062873840332, + "loss": 0.226, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -6.7219719886779785, + "rewards/margins": 7.483970642089844, + "rewards/rejected": -14.20594310760498, + "step": 1490 + }, + { + "epoch": 1.3321492007104796, + "grad_norm": 3.039519786834717, + "learning_rate": 0.0, + "logits/chosen": 2.8715057373046875, + "logits/rejected": 2.4751758575439453, + "logps/chosen": -4.350632667541504, + "logps/rejected": -9.331658363342285, + "loss": 0.1691, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -6.525949001312256, + "rewards/margins": 7.471539497375488, + "rewards/rejected": -13.997488021850586, + "step": 1500 + }, + { + "epoch": 1.3321492007104796, + "eval_logits/chosen": 2.7312934398651123, + "eval_logits/rejected": 2.52182936668396, + "eval_logps/chosen": -4.376524448394775, + "eval_logps/rejected": -8.91296672821045, + "eval_loss": 0.20589081943035126, + "eval_rewards/accuracies": 0.9780219793319702, + "eval_rewards/chosen": -6.564786911010742, + "eval_rewards/margins": 6.804664134979248, + "eval_rewards/rejected": -13.369450569152832, + "eval_runtime": 25.2592, + "eval_samples_per_second": 28.821, + "eval_steps_per_second": 3.603, + "step": 1500 + } + ], + "logging_steps": 10, + "max_steps": 1500, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 3.531829928376926e+18, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}