| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.24053333333333332, | |
| "eval_steps": 500, | |
| "global_step": 451, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 503.5833492279053, | |
| "epoch": 0.0010666666666666667, | |
| "grad_norm": 0.13479497816516098, | |
| "kl": 0.0, | |
| "learning_rate": 7.142857142857142e-08, | |
| "loss": -0.0, | |
| "reward": 0.291666672565043, | |
| "reward_std": 0.4538230746984482, | |
| "rewards/equation_reward_func": 0.026041667442768812, | |
| "rewards/format_reward_func": 0.2656250074505806, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 508.4817867279053, | |
| "epoch": 0.0021333333333333334, | |
| "grad_norm": 0.12733498910966365, | |
| "kl": 0.0003889799118041992, | |
| "learning_rate": 1.4285714285714285e-07, | |
| "loss": 0.0, | |
| "reward": 0.33333334047347307, | |
| "reward_std": 0.46788670495152473, | |
| "rewards/equation_reward_func": 0.057291667675599456, | |
| "rewards/format_reward_func": 0.27604167349636555, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 491.1354351043701, | |
| "epoch": 0.0032, | |
| "grad_norm": 0.12351612192306213, | |
| "kl": 0.0004049539566040039, | |
| "learning_rate": 2.1428571428571426e-07, | |
| "loss": 0.0, | |
| "reward": 0.29427083767950535, | |
| "reward_std": 0.45141119323670864, | |
| "rewards/equation_reward_func": 0.05468750186264515, | |
| "rewards/format_reward_func": 0.2395833395421505, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 519.4869918823242, | |
| "epoch": 0.004266666666666667, | |
| "grad_norm": 0.13325656620448842, | |
| "kl": 0.00039446353912353516, | |
| "learning_rate": 2.857142857142857e-07, | |
| "loss": 0.0, | |
| "reward": 0.32812500931322575, | |
| "reward_std": 0.48463437519967556, | |
| "rewards/equation_reward_func": 0.03645833418704569, | |
| "rewards/format_reward_func": 0.291666672565043, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 494.9791774749756, | |
| "epoch": 0.005333333333333333, | |
| "grad_norm": 0.13179701975607572, | |
| "kl": 0.00047516822814941406, | |
| "learning_rate": 3.5714285714285716e-07, | |
| "loss": 0.0, | |
| "reward": 0.3489583423361182, | |
| "reward_std": 0.46488506719470024, | |
| "rewards/equation_reward_func": 0.05468750116415322, | |
| "rewards/format_reward_func": 0.29427084140479565, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 494.3021011352539, | |
| "epoch": 0.0064, | |
| "grad_norm": 0.14132727969735295, | |
| "kl": 0.0004602670669555664, | |
| "learning_rate": 4.285714285714285e-07, | |
| "loss": 0.0, | |
| "reward": 0.31770834419876337, | |
| "reward_std": 0.48818780295550823, | |
| "rewards/equation_reward_func": 0.07552083465270698, | |
| "rewards/format_reward_func": 0.24218750838190317, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 475.71876335144043, | |
| "epoch": 0.007466666666666667, | |
| "grad_norm": 0.11970698727031565, | |
| "kl": 0.0008485317230224609, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0, | |
| "reward": 0.45833334513008595, | |
| "reward_std": 0.5431181341409683, | |
| "rewards/equation_reward_func": 0.06770833441987634, | |
| "rewards/format_reward_func": 0.39062500558793545, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 473.8307399749756, | |
| "epoch": 0.008533333333333334, | |
| "grad_norm": 0.12821306696149545, | |
| "kl": 0.001085042953491211, | |
| "learning_rate": 4.999740409224932e-07, | |
| "loss": 0.0, | |
| "reward": 0.5078125167638063, | |
| "reward_std": 0.5308652929961681, | |
| "rewards/equation_reward_func": 0.07031250186264515, | |
| "rewards/format_reward_func": 0.4375000149011612, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 463.0729293823242, | |
| "epoch": 0.0096, | |
| "grad_norm": 0.1086684955444935, | |
| "kl": 0.0031414031982421875, | |
| "learning_rate": 4.998961690809627e-07, | |
| "loss": 0.0, | |
| "reward": 0.6588541828095913, | |
| "reward_std": 0.5097422748804092, | |
| "rewards/equation_reward_func": 0.05468750139698386, | |
| "rewards/format_reward_func": 0.6041666902601719, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 478.442720413208, | |
| "epoch": 0.010666666666666666, | |
| "grad_norm": 0.11631333461599484, | |
| "kl": 0.00470733642578125, | |
| "learning_rate": 4.997664006472578e-07, | |
| "loss": 0.0, | |
| "reward": 0.7031250204890966, | |
| "reward_std": 0.49558274634182453, | |
| "rewards/equation_reward_func": 0.049479167675599456, | |
| "rewards/format_reward_func": 0.6536458525806665, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 477.4974117279053, | |
| "epoch": 0.011733333333333333, | |
| "grad_norm": 0.10506512614686118, | |
| "kl": 0.00562286376953125, | |
| "learning_rate": 4.995847625707292e-07, | |
| "loss": 0.0, | |
| "reward": 0.802083358168602, | |
| "reward_std": 0.4941887054592371, | |
| "rewards/equation_reward_func": 0.06510416814126074, | |
| "rewards/format_reward_func": 0.7369791939854622, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 467.4010543823242, | |
| "epoch": 0.0128, | |
| "grad_norm": 0.08192641981472143, | |
| "kl": 0.005817413330078125, | |
| "learning_rate": 4.993512925726318e-07, | |
| "loss": 0.0, | |
| "reward": 0.8828125186264515, | |
| "reward_std": 0.4274959657341242, | |
| "rewards/equation_reward_func": 0.0911458358168602, | |
| "rewards/format_reward_func": 0.7916666865348816, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 455.14063453674316, | |
| "epoch": 0.013866666666666666, | |
| "grad_norm": 0.09142328720580743, | |
| "kl": 0.0064449310302734375, | |
| "learning_rate": 4.990660391382923e-07, | |
| "loss": 0.0, | |
| "reward": 0.9062500260770321, | |
| "reward_std": 0.4164346093311906, | |
| "rewards/equation_reward_func": 0.07552083441987634, | |
| "rewards/format_reward_func": 0.8307291865348816, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 467.2135543823242, | |
| "epoch": 0.014933333333333333, | |
| "grad_norm": 0.07919215402797099, | |
| "kl": 0.008056640625, | |
| "learning_rate": 4.987290615070384e-07, | |
| "loss": 0.0, | |
| "reward": 0.9062500260770321, | |
| "reward_std": 0.3465987662784755, | |
| "rewards/equation_reward_func": 0.04947916720993817, | |
| "rewards/format_reward_func": 0.8567708544433117, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 470.7395935058594, | |
| "epoch": 0.016, | |
| "grad_norm": 0.08287581891178063, | |
| "kl": 0.009038925170898438, | |
| "learning_rate": 4.983404296598978e-07, | |
| "loss": 0.0, | |
| "reward": 0.9635416977107525, | |
| "reward_std": 0.37379608815535903, | |
| "rewards/equation_reward_func": 0.08333333604969084, | |
| "rewards/format_reward_func": 0.8802083432674408, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 467.1510543823242, | |
| "epoch": 0.017066666666666667, | |
| "grad_norm": 0.06256565056632399, | |
| "kl": 0.0106964111328125, | |
| "learning_rate": 4.979002243050646e-07, | |
| "loss": 0.0, | |
| "reward": 1.0000000298023224, | |
| "reward_std": 0.3002084447070956, | |
| "rewards/equation_reward_func": 0.08072916883975267, | |
| "rewards/format_reward_func": 0.9192708507180214, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 443.64845275878906, | |
| "epoch": 0.018133333333333335, | |
| "grad_norm": 0.07994625189461813, | |
| "kl": 0.014324188232421875, | |
| "learning_rate": 4.974085368611381e-07, | |
| "loss": 0.0, | |
| "reward": 1.0442708767950535, | |
| "reward_std": 0.3554811980575323, | |
| "rewards/equation_reward_func": 0.12239583558402956, | |
| "rewards/format_reward_func": 0.9218750149011612, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 405.1770935058594, | |
| "epoch": 0.0192, | |
| "grad_norm": 0.07683793642034875, | |
| "kl": 0.013385772705078125, | |
| "learning_rate": 4.968654694381379e-07, | |
| "loss": 0.0, | |
| "reward": 1.0885417014360428, | |
| "reward_std": 0.28349386202171445, | |
| "rewards/equation_reward_func": 0.12239583604969084, | |
| "rewards/format_reward_func": 0.9661458507180214, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 418.44532012939453, | |
| "epoch": 0.020266666666666665, | |
| "grad_norm": 0.07890389062948255, | |
| "kl": 0.0151214599609375, | |
| "learning_rate": 4.962711348162987e-07, | |
| "loss": 0.0, | |
| "reward": 1.0859375335276127, | |
| "reward_std": 0.3367053628899157, | |
| "rewards/equation_reward_func": 0.13541667046956718, | |
| "rewards/format_reward_func": 0.9505208507180214, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 434.3203239440918, | |
| "epoch": 0.021333333333333333, | |
| "grad_norm": 0.06757013446913575, | |
| "kl": 0.014007568359375, | |
| "learning_rate": 4.956256564226487e-07, | |
| "loss": 0.0, | |
| "reward": 1.0520833805203438, | |
| "reward_std": 0.23739226441830397, | |
| "rewards/equation_reward_func": 0.0989583358168602, | |
| "rewards/format_reward_func": 0.9531250149011612, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 429.4140739440918, | |
| "epoch": 0.0224, | |
| "grad_norm": 0.06720599754456791, | |
| "kl": 0.0153350830078125, | |
| "learning_rate": 4.949291683053768e-07, | |
| "loss": 0.0, | |
| "reward": 1.0442708730697632, | |
| "reward_std": 0.31464048475027084, | |
| "rewards/equation_reward_func": 0.10677083628252149, | |
| "rewards/format_reward_func": 0.9375000149011612, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 393.57552909851074, | |
| "epoch": 0.023466666666666667, | |
| "grad_norm": 0.05753675162494118, | |
| "kl": 0.017383575439453125, | |
| "learning_rate": 4.941818151059955e-07, | |
| "loss": 0.0, | |
| "reward": 1.0598958507180214, | |
| "reward_std": 0.19039153354242444, | |
| "rewards/equation_reward_func": 0.08072916767559946, | |
| "rewards/format_reward_func": 0.979166679084301, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 402.08594512939453, | |
| "epoch": 0.024533333333333334, | |
| "grad_norm": 0.08104644552589652, | |
| "kl": 0.016567230224609375, | |
| "learning_rate": 4.933837520293017e-07, | |
| "loss": 0.0, | |
| "reward": 1.0703125447034836, | |
| "reward_std": 0.31082099210470915, | |
| "rewards/equation_reward_func": 0.11979167046956718, | |
| "rewards/format_reward_func": 0.9505208507180214, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 392.38802909851074, | |
| "epoch": 0.0256, | |
| "grad_norm": 0.08134478820611701, | |
| "kl": 0.017536163330078125, | |
| "learning_rate": 4.925351448111454e-07, | |
| "loss": 0.0, | |
| "reward": 1.096354205161333, | |
| "reward_std": 0.2948869550600648, | |
| "rewards/equation_reward_func": 0.1302083353511989, | |
| "rewards/format_reward_func": 0.9661458432674408, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 402.7161560058594, | |
| "epoch": 0.02666666666666667, | |
| "grad_norm": 0.07675619360564714, | |
| "kl": 0.01970672607421875, | |
| "learning_rate": 4.91636169684011e-07, | |
| "loss": 0.0, | |
| "reward": 1.0989583618938923, | |
| "reward_std": 0.27427004277706146, | |
| "rewards/equation_reward_func": 0.13020833767950535, | |
| "rewards/format_reward_func": 0.9687500186264515, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 397.1432418823242, | |
| "epoch": 0.027733333333333332, | |
| "grad_norm": 0.08739757866005847, | |
| "kl": 0.021942138671875, | |
| "learning_rate": 4.906870133404186e-07, | |
| "loss": 0.0, | |
| "reward": 1.101562537252903, | |
| "reward_std": 0.2725550555624068, | |
| "rewards/equation_reward_func": 0.13281250232830644, | |
| "rewards/format_reward_func": 0.9687500223517418, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 385.5911560058594, | |
| "epoch": 0.0288, | |
| "grad_norm": 0.07937191906586637, | |
| "kl": 0.0219573974609375, | |
| "learning_rate": 4.896878728941531e-07, | |
| "loss": 0.0, | |
| "reward": 1.1067708656191826, | |
| "reward_std": 0.23853057762607932, | |
| "rewards/equation_reward_func": 0.1328125020954758, | |
| "rewards/format_reward_func": 0.9739583432674408, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 390.7760486602783, | |
| "epoch": 0.029866666666666666, | |
| "grad_norm": 0.0867908268216988, | |
| "kl": 0.02423858642578125, | |
| "learning_rate": 4.886389558393284e-07, | |
| "loss": 0.0, | |
| "reward": 1.1171875447034836, | |
| "reward_std": 0.26454683812335134, | |
| "rewards/equation_reward_func": 0.13802083814516664, | |
| "rewards/format_reward_func": 0.979166679084301, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 380.88282203674316, | |
| "epoch": 0.030933333333333334, | |
| "grad_norm": 0.08529999627698764, | |
| "kl": 0.0235748291015625, | |
| "learning_rate": 4.875404800072976e-07, | |
| "loss": 0.0, | |
| "reward": 1.1041666939854622, | |
| "reward_std": 0.2771674753166735, | |
| "rewards/equation_reward_func": 0.13541667046956718, | |
| "rewards/format_reward_func": 0.9687500223517418, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 353.28646659851074, | |
| "epoch": 0.032, | |
| "grad_norm": 0.09143043966386487, | |
| "kl": 0.028717041015625, | |
| "learning_rate": 4.86392673521415e-07, | |
| "loss": 0.0, | |
| "reward": 1.1171875298023224, | |
| "reward_std": 0.24405073188245296, | |
| "rewards/equation_reward_func": 0.13020833721384406, | |
| "rewards/format_reward_func": 0.986979179084301, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 356.72917556762695, | |
| "epoch": 0.03306666666666667, | |
| "grad_norm": 0.07996321279906737, | |
| "kl": 0.02960205078125, | |
| "learning_rate": 4.851957747496606e-07, | |
| "loss": 0.0, | |
| "reward": 1.1067708730697632, | |
| "reward_std": 0.271460447460413, | |
| "rewards/equation_reward_func": 0.1302083374466747, | |
| "rewards/format_reward_func": 0.9765625111758709, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 359.3958435058594, | |
| "epoch": 0.034133333333333335, | |
| "grad_norm": 0.07947666131932718, | |
| "kl": 0.03427886962890625, | |
| "learning_rate": 4.839500322551386e-07, | |
| "loss": 0.0, | |
| "reward": 1.0729166977107525, | |
| "reward_std": 0.182542169932276, | |
| "rewards/equation_reward_func": 0.09635417093522847, | |
| "rewards/format_reward_func": 0.9765625149011612, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 336.5026111602783, | |
| "epoch": 0.0352, | |
| "grad_norm": 0.08769515962748972, | |
| "kl": 0.03688812255859375, | |
| "learning_rate": 4.826557047444563e-07, | |
| "loss": 0.0, | |
| "reward": 1.1380208730697632, | |
| "reward_std": 0.22655905643478036, | |
| "rewards/equation_reward_func": 0.14843750442378223, | |
| "rewards/format_reward_func": 0.9895833432674408, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 342.29427909851074, | |
| "epoch": 0.03626666666666667, | |
| "grad_norm": 0.086217122830427, | |
| "kl": 0.0422821044921875, | |
| "learning_rate": 4.813130610139993e-07, | |
| "loss": 0.0, | |
| "reward": 1.1744792088866234, | |
| "reward_std": 0.2655471176840365, | |
| "rewards/equation_reward_func": 0.18750000768341124, | |
| "rewards/format_reward_func": 0.986979179084301, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 334.3255271911621, | |
| "epoch": 0.037333333333333336, | |
| "grad_norm": 0.09173810114071636, | |
| "kl": 0.0477447509765625, | |
| "learning_rate": 4.799223798941089e-07, | |
| "loss": 0.0, | |
| "reward": 1.1562500409781933, | |
| "reward_std": 0.26141493022441864, | |
| "rewards/equation_reward_func": 0.1718750074505806, | |
| "rewards/format_reward_func": 0.9843750111758709, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 344.3619861602783, | |
| "epoch": 0.0384, | |
| "grad_norm": 0.07105320252206761, | |
| "kl": 0.046356201171875, | |
| "learning_rate": 4.78483950191177e-07, | |
| "loss": 0.0, | |
| "reward": 1.1484375298023224, | |
| "reward_std": 0.2299756333231926, | |
| "rewards/equation_reward_func": 0.16145833721384406, | |
| "rewards/format_reward_func": 0.9869791716337204, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 331.7369909286499, | |
| "epoch": 0.039466666666666664, | |
| "grad_norm": 0.09213749486752489, | |
| "kl": 0.0506744384765625, | |
| "learning_rate": 4.769980706276687e-07, | |
| "loss": 0.0001, | |
| "reward": 1.1796875298023224, | |
| "reward_std": 0.3097896152175963, | |
| "rewards/equation_reward_func": 0.20312500442378223, | |
| "rewards/format_reward_func": 0.9765625186264515, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 317.6041793823242, | |
| "epoch": 0.04053333333333333, | |
| "grad_norm": 0.0753029614601889, | |
| "kl": 0.059661865234375, | |
| "learning_rate": 4.7546504978008595e-07, | |
| "loss": 0.0001, | |
| "reward": 1.1718750447034836, | |
| "reward_std": 0.2521190056577325, | |
| "rewards/equation_reward_func": 0.18229167140088975, | |
| "rewards/format_reward_func": 0.9895833395421505, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 319.7109441757202, | |
| "epoch": 0.0416, | |
| "grad_norm": 0.10940993762984497, | |
| "kl": 0.0585174560546875, | |
| "learning_rate": 4.738852060148848e-07, | |
| "loss": 0.0001, | |
| "reward": 1.257812537252903, | |
| "reward_std": 0.29705953504890203, | |
| "rewards/equation_reward_func": 0.2656250062864274, | |
| "rewards/format_reward_func": 0.9921875074505806, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 314.92709827423096, | |
| "epoch": 0.042666666666666665, | |
| "grad_norm": 0.09384920739685453, | |
| "kl": 0.06884765625, | |
| "learning_rate": 4.722588674223593e-07, | |
| "loss": 0.0001, | |
| "reward": 1.1875000298023224, | |
| "reward_std": 0.27663656743243337, | |
| "rewards/equation_reward_func": 0.20052083930931985, | |
| "rewards/format_reward_func": 0.986979179084301, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 330.8880281448364, | |
| "epoch": 0.04373333333333333, | |
| "grad_norm": 0.08954239890220438, | |
| "kl": 0.06683349609375, | |
| "learning_rate": 4.70586371748506e-07, | |
| "loss": 0.0001, | |
| "reward": 1.177083358168602, | |
| "reward_std": 0.2589058754965663, | |
| "rewards/equation_reward_func": 0.1901041695382446, | |
| "rewards/format_reward_func": 0.986979179084301, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 325.97657203674316, | |
| "epoch": 0.0448, | |
| "grad_norm": 0.07613263304745997, | |
| "kl": 0.074432373046875, | |
| "learning_rate": 4.6886806632488363e-07, | |
| "loss": 0.0001, | |
| "reward": 1.1953125298023224, | |
| "reward_std": 0.2278480869717896, | |
| "rewards/equation_reward_func": 0.19791667396202683, | |
| "rewards/format_reward_func": 0.9973958358168602, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 303.3281354904175, | |
| "epoch": 0.04586666666666667, | |
| "grad_norm": 0.07897352847654085, | |
| "kl": 0.0804290771484375, | |
| "learning_rate": 4.6710430799648143e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2083333730697632, | |
| "reward_std": 0.24759106524288654, | |
| "rewards/equation_reward_func": 0.21354167256504297, | |
| "rewards/format_reward_func": 0.9947916679084301, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 329.1692819595337, | |
| "epoch": 0.046933333333333334, | |
| "grad_norm": 0.09548085778824623, | |
| "kl": 0.07666015625, | |
| "learning_rate": 4.652954630476127e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2447916939854622, | |
| "reward_std": 0.2977108689956367, | |
| "rewards/equation_reward_func": 0.2578125041909516, | |
| "rewards/format_reward_func": 0.986979179084301, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 336.86719703674316, | |
| "epoch": 0.048, | |
| "grad_norm": 0.09182200209153558, | |
| "kl": 0.0791778564453125, | |
| "learning_rate": 4.6344190712584713e-07, | |
| "loss": 0.0001, | |
| "reward": 1.174479216337204, | |
| "reward_std": 0.19297246867790818, | |
| "rewards/equation_reward_func": 0.17968750442378223, | |
| "rewards/format_reward_func": 0.9947916716337204, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 302.16928005218506, | |
| "epoch": 0.04906666666666667, | |
| "grad_norm": 0.06795618609015738, | |
| "kl": 0.0859375, | |
| "learning_rate": 4.615440251639995e-07, | |
| "loss": 0.0001, | |
| "reward": 1.1718750447034836, | |
| "reward_std": 0.22476374451071024, | |
| "rewards/equation_reward_func": 0.17968750465661287, | |
| "rewards/format_reward_func": 0.9921875074505806, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 310.22917461395264, | |
| "epoch": 0.050133333333333335, | |
| "grad_norm": 0.10599200724175652, | |
| "kl": 0.1083984375, | |
| "learning_rate": 4.596022113001894e-07, | |
| "loss": 0.0001, | |
| "reward": 1.250000037252903, | |
| "reward_std": 0.22750707995146513, | |
| "rewards/equation_reward_func": 0.2630208421032876, | |
| "rewards/format_reward_func": 0.986979179084301, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 329.9765748977661, | |
| "epoch": 0.0512, | |
| "grad_norm": 0.0863029718400734, | |
| "kl": 0.08917236328125, | |
| "learning_rate": 4.576168687959895e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2083333730697632, | |
| "reward_std": 0.3378387549892068, | |
| "rewards/equation_reward_func": 0.23437500512227416, | |
| "rewards/format_reward_func": 0.9739583507180214, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 327.7343807220459, | |
| "epoch": 0.05226666666666667, | |
| "grad_norm": 0.07908824639225548, | |
| "kl": 0.090667724609375, | |
| "learning_rate": 4.555884099526793e-07, | |
| "loss": 0.0001, | |
| "reward": 1.1562500298023224, | |
| "reward_std": 0.22620543837547302, | |
| "rewards/equation_reward_func": 0.16927083767950535, | |
| "rewards/format_reward_func": 0.9869791753590107, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 301.43230533599854, | |
| "epoch": 0.05333333333333334, | |
| "grad_norm": 0.08186277709006264, | |
| "kl": 0.091705322265625, | |
| "learning_rate": 4.5351725602562174e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2135416939854622, | |
| "reward_std": 0.2680189316160977, | |
| "rewards/equation_reward_func": 0.22135417256504297, | |
| "rewards/format_reward_func": 0.9921875074505806, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 311.4296941757202, | |
| "epoch": 0.0544, | |
| "grad_norm": 0.11220927118789871, | |
| "kl": 0.091064453125, | |
| "learning_rate": 4.514038371367791e-07, | |
| "loss": 0.0001, | |
| "reward": 1.218750037252903, | |
| "reward_std": 0.2762691476382315, | |
| "rewards/equation_reward_func": 0.23697917559184134, | |
| "rewards/format_reward_func": 0.9817708469927311, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 309.2500104904175, | |
| "epoch": 0.055466666666666664, | |
| "grad_norm": 0.09377829405226727, | |
| "kl": 0.0958251953125, | |
| "learning_rate": 4.4924859218538936e-07, | |
| "loss": 0.0001, | |
| "reward": 1.276041705161333, | |
| "reward_std": 0.27660042280331254, | |
| "rewards/equation_reward_func": 0.28125000838190317, | |
| "rewards/format_reward_func": 0.9947916679084301, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 348.30209159851074, | |
| "epoch": 0.05653333333333333, | |
| "grad_norm": 0.07803762538430126, | |
| "kl": 0.095245361328125, | |
| "learning_rate": 4.470519687568185e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2239583805203438, | |
| "reward_std": 0.2814735807478428, | |
| "rewards/equation_reward_func": 0.2500000086147338, | |
| "rewards/format_reward_func": 0.9739583469927311, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 314.0182361602783, | |
| "epoch": 0.0576, | |
| "grad_norm": 0.10165220890741065, | |
| "kl": 0.09869384765625, | |
| "learning_rate": 4.4481442302960923e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2213542014360428, | |
| "reward_std": 0.30849385308101773, | |
| "rewards/equation_reward_func": 0.24479167186655104, | |
| "rewards/format_reward_func": 0.9765625111758709, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 321.40625953674316, | |
| "epoch": 0.058666666666666666, | |
| "grad_norm": 0.10983940687806108, | |
| "kl": 0.103790283203125, | |
| "learning_rate": 4.4253641968074505e-07, | |
| "loss": 0.0001, | |
| "reward": 1.242187537252903, | |
| "reward_std": 0.2469061641022563, | |
| "rewards/equation_reward_func": 0.25781250931322575, | |
| "rewards/format_reward_func": 0.9843750149011612, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 316.86459827423096, | |
| "epoch": 0.05973333333333333, | |
| "grad_norm": 0.09985333306751591, | |
| "kl": 0.091156005859375, | |
| "learning_rate": 4.402184317891501e-07, | |
| "loss": 0.0001, | |
| "reward": 1.317708358168602, | |
| "reward_std": 0.31702343560755253, | |
| "rewards/equation_reward_func": 0.33333334140479565, | |
| "rewards/format_reward_func": 0.9843750074505806, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 328.5885543823242, | |
| "epoch": 0.0608, | |
| "grad_norm": 0.1106376946803666, | |
| "kl": 0.0948486328125, | |
| "learning_rate": 4.37860940737443e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3125000335276127, | |
| "reward_std": 0.3574067768640816, | |
| "rewards/equation_reward_func": 0.33854167722165585, | |
| "rewards/format_reward_func": 0.9739583507180214, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 363.8151149749756, | |
| "epoch": 0.06186666666666667, | |
| "grad_norm": 0.09009352516811832, | |
| "kl": 0.08758544921875, | |
| "learning_rate": 4.354644361119671e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2291667088866234, | |
| "reward_std": 0.30905032297596335, | |
| "rewards/equation_reward_func": 0.2578125116415322, | |
| "rewards/format_reward_func": 0.971354179084301, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 339.27344703674316, | |
| "epoch": 0.06293333333333333, | |
| "grad_norm": 0.10917265236088759, | |
| "kl": 0.0970458984375, | |
| "learning_rate": 4.3302941560111716e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3229167014360428, | |
| "reward_std": 0.3481226065196097, | |
| "rewards/equation_reward_func": 0.3437500095460564, | |
| "rewards/format_reward_func": 0.9791666828095913, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 346.77084159851074, | |
| "epoch": 0.064, | |
| "grad_norm": 0.08327353982344886, | |
| "kl": 0.097930908203125, | |
| "learning_rate": 4.3055638489198236e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2369792014360428, | |
| "reward_std": 0.2683186633512378, | |
| "rewards/equation_reward_func": 0.2578125090803951, | |
| "rewards/format_reward_func": 0.979166679084301, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 337.7552185058594, | |
| "epoch": 0.06506666666666666, | |
| "grad_norm": 0.10607954735968793, | |
| "kl": 0.102020263671875, | |
| "learning_rate": 4.280458575653296e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2864583805203438, | |
| "reward_std": 0.27850370202213526, | |
| "rewards/equation_reward_func": 0.3020833437331021, | |
| "rewards/format_reward_func": 0.9843750111758709, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 370.64323806762695, | |
| "epoch": 0.06613333333333334, | |
| "grad_norm": 0.083470198281649, | |
| "kl": 0.1005859375, | |
| "learning_rate": 4.2549835498894665e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2656250298023224, | |
| "reward_std": 0.24806254683062434, | |
| "rewards/equation_reward_func": 0.28385417140088975, | |
| "rewards/format_reward_func": 0.9817708432674408, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 367.3099021911621, | |
| "epoch": 0.0672, | |
| "grad_norm": 0.08670778520411884, | |
| "kl": 0.10858154296875, | |
| "learning_rate": 4.229144062093679e-07, | |
| "loss": 0.0001, | |
| "reward": 1.229166705161333, | |
| "reward_std": 0.28506570123136044, | |
| "rewards/equation_reward_func": 0.25000000512227416, | |
| "rewards/format_reward_func": 0.9791666828095913, | |
| "step": 126 | |
| }, | |
| { | |
| "completion_length": 367.12501335144043, | |
| "epoch": 0.06826666666666667, | |
| "grad_norm": 0.09682219451984418, | |
| "kl": 0.107513427734375, | |
| "learning_rate": 4.2029454784200675e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2916667014360428, | |
| "reward_std": 0.30678933998569846, | |
| "rewards/equation_reward_func": 0.30989584419876337, | |
| "rewards/format_reward_func": 0.9817708469927311, | |
| "step": 128 | |
| }, | |
| { | |
| "completion_length": 360.00001525878906, | |
| "epoch": 0.06933333333333333, | |
| "grad_norm": 0.09090015857191287, | |
| "kl": 0.1029052734375, | |
| "learning_rate": 4.1763932395971433e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3020833656191826, | |
| "reward_std": 0.25149599462747574, | |
| "rewards/equation_reward_func": 0.31770834047347307, | |
| "rewards/format_reward_func": 0.9843750111758709, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 360.1640739440918, | |
| "epoch": 0.0704, | |
| "grad_norm": 0.07789416337575122, | |
| "kl": 0.108978271484375, | |
| "learning_rate": 4.1494928597979117e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3020833730697632, | |
| "reward_std": 0.2967960089445114, | |
| "rewards/equation_reward_func": 0.33333334093913436, | |
| "rewards/format_reward_func": 0.9687500149011612, | |
| "step": 132 | |
| }, | |
| { | |
| "completion_length": 371.6197986602783, | |
| "epoch": 0.07146666666666666, | |
| "grad_norm": 0.07446852162437392, | |
| "kl": 0.13824462890625, | |
| "learning_rate": 4.122249925494726e-07, | |
| "loss": 0.0001, | |
| "reward": 1.341145884245634, | |
| "reward_std": 0.25496263802051544, | |
| "rewards/equation_reward_func": 0.3593750118743628, | |
| "rewards/format_reward_func": 0.9817708469927311, | |
| "step": 134 | |
| }, | |
| { | |
| "completion_length": 353.9765739440918, | |
| "epoch": 0.07253333333333334, | |
| "grad_norm": 0.0907449153165434, | |
| "kl": 0.111083984375, | |
| "learning_rate": 4.094670094299131e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2994792014360428, | |
| "reward_std": 0.2716127992607653, | |
| "rewards/equation_reward_func": 0.31510417722165585, | |
| "rewards/format_reward_func": 0.9843750149011612, | |
| "step": 136 | |
| }, | |
| { | |
| "completion_length": 385.434907913208, | |
| "epoch": 0.0736, | |
| "grad_norm": 0.09757570719773351, | |
| "kl": 0.118072509765625, | |
| "learning_rate": 4.066759093786931e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2447916939854622, | |
| "reward_std": 0.27319318102672696, | |
| "rewards/equation_reward_func": 0.2786458386108279, | |
| "rewards/format_reward_func": 0.9661458507180214, | |
| "step": 138 | |
| }, | |
| { | |
| "completion_length": 364.78646755218506, | |
| "epoch": 0.07466666666666667, | |
| "grad_norm": 0.09303261300149815, | |
| "kl": 0.1240234375, | |
| "learning_rate": 4.038522720308732e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3619792088866234, | |
| "reward_std": 0.2764986059628427, | |
| "rewards/equation_reward_func": 0.39322918094694614, | |
| "rewards/format_reward_func": 0.9687500186264515, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 381.4687604904175, | |
| "epoch": 0.07573333333333333, | |
| "grad_norm": 0.10078837483196365, | |
| "kl": 0.114532470703125, | |
| "learning_rate": 4.009966837786194e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2942708656191826, | |
| "reward_std": 0.2905842326581478, | |
| "rewards/equation_reward_func": 0.32291667349636555, | |
| "rewards/format_reward_func": 0.971354179084301, | |
| "step": 142 | |
| }, | |
| { | |
| "completion_length": 365.0182361602783, | |
| "epoch": 0.0768, | |
| "grad_norm": 0.09451735002608201, | |
| "kl": 0.134368896484375, | |
| "learning_rate": 3.981097376494259e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3723958730697632, | |
| "reward_std": 0.228050971403718, | |
| "rewards/equation_reward_func": 0.37760417512618005, | |
| "rewards/format_reward_func": 0.9947916716337204, | |
| "step": 144 | |
| }, | |
| { | |
| "completion_length": 381.89844703674316, | |
| "epoch": 0.07786666666666667, | |
| "grad_norm": 0.09710751540038805, | |
| "kl": 0.121063232421875, | |
| "learning_rate": 3.951920331829592e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3020833767950535, | |
| "reward_std": 0.35034706676378846, | |
| "rewards/equation_reward_func": 0.3593750086147338, | |
| "rewards/format_reward_func": 0.9427083469927311, | |
| "step": 146 | |
| }, | |
| { | |
| "completion_length": 360.5416774749756, | |
| "epoch": 0.07893333333333333, | |
| "grad_norm": 0.1060852558502134, | |
| "kl": 0.137664794921875, | |
| "learning_rate": 3.922441763065506e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3802083730697632, | |
| "reward_std": 0.30321967136114836, | |
| "rewards/equation_reward_func": 0.4114583428017795, | |
| "rewards/format_reward_func": 0.9687500186264515, | |
| "step": 148 | |
| }, | |
| { | |
| "completion_length": 388.96355056762695, | |
| "epoch": 0.08, | |
| "grad_norm": 0.07982357000644656, | |
| "kl": 0.13201904296875, | |
| "learning_rate": 3.8926677920936093e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3177083656191826, | |
| "reward_std": 0.28983199410140514, | |
| "rewards/equation_reward_func": 0.35677084559574723, | |
| "rewards/format_reward_func": 0.9609375149011612, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 358.716157913208, | |
| "epoch": 0.08106666666666666, | |
| "grad_norm": 0.10617667545751451, | |
| "kl": 0.15240478515625, | |
| "learning_rate": 3.862604602152464e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4088542088866234, | |
| "reward_std": 0.3088458301499486, | |
| "rewards/equation_reward_func": 0.4375000139698386, | |
| "rewards/format_reward_func": 0.971354179084301, | |
| "step": 152 | |
| }, | |
| { | |
| "completion_length": 391.22657012939453, | |
| "epoch": 0.08213333333333334, | |
| "grad_norm": 0.07877140143109437, | |
| "kl": 0.157135009765625, | |
| "learning_rate": 3.8322584365434934e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3984375298023224, | |
| "reward_std": 0.32617295952513814, | |
| "rewards/equation_reward_func": 0.44010418467223644, | |
| "rewards/format_reward_func": 0.9583333544433117, | |
| "step": 154 | |
| }, | |
| { | |
| "completion_length": 375.8619918823242, | |
| "epoch": 0.0832, | |
| "grad_norm": 0.09944822233149916, | |
| "kl": 0.143798828125, | |
| "learning_rate": 3.8016355973344173e-07, | |
| "loss": 0.0001, | |
| "reward": 1.395833384245634, | |
| "reward_std": 0.3326970827765763, | |
| "rewards/equation_reward_func": 0.4427083448972553, | |
| "rewards/format_reward_func": 0.9531250074505806, | |
| "step": 156 | |
| }, | |
| { | |
| "completion_length": 406.1145935058594, | |
| "epoch": 0.08426666666666667, | |
| "grad_norm": 0.11629394234661612, | |
| "kl": 0.152587890625, | |
| "learning_rate": 3.7707424440504863e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3411458805203438, | |
| "reward_std": 0.36975874193012714, | |
| "rewards/equation_reward_func": 0.3906250095460564, | |
| "rewards/format_reward_func": 0.9505208544433117, | |
| "step": 158 | |
| }, | |
| { | |
| "completion_length": 357.35938453674316, | |
| "epoch": 0.08533333333333333, | |
| "grad_norm": 0.07541836663359057, | |
| "kl": 0.170257568359375, | |
| "learning_rate": 3.739585392353787e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4270833730697632, | |
| "reward_std": 0.21465489408001304, | |
| "rewards/equation_reward_func": 0.4505208507180214, | |
| "rewards/format_reward_func": 0.9765625149011612, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 350.80990505218506, | |
| "epoch": 0.0864, | |
| "grad_norm": 0.07225346641457608, | |
| "kl": 0.158721923828125, | |
| "learning_rate": 3.7081709127108767e-07, | |
| "loss": 0.0002, | |
| "reward": 1.484375037252903, | |
| "reward_std": 0.2267396138049662, | |
| "rewards/equation_reward_func": 0.520833347691223, | |
| "rewards/format_reward_func": 0.9635416828095913, | |
| "step": 162 | |
| }, | |
| { | |
| "completion_length": 419.6380310058594, | |
| "epoch": 0.08746666666666666, | |
| "grad_norm": 0.08117087970991471, | |
| "kl": 0.13812255859375, | |
| "learning_rate": 3.6765055290490513e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3046875484287739, | |
| "reward_std": 0.28110535256564617, | |
| "rewards/equation_reward_func": 0.3567708432674408, | |
| "rewards/format_reward_func": 0.947916679084301, | |
| "step": 164 | |
| }, | |
| { | |
| "completion_length": 427.7942810058594, | |
| "epoch": 0.08853333333333334, | |
| "grad_norm": 0.08133164384186187, | |
| "kl": 0.139892578125, | |
| "learning_rate": 3.644595817401501e-07, | |
| "loss": 0.0001, | |
| "reward": 1.278645884245634, | |
| "reward_std": 0.28350870544090867, | |
| "rewards/equation_reward_func": 0.3255208416376263, | |
| "rewards/format_reward_func": 0.9531250186264515, | |
| "step": 166 | |
| }, | |
| { | |
| "completion_length": 423.0573043823242, | |
| "epoch": 0.0896, | |
| "grad_norm": 0.08618132733280144, | |
| "kl": 0.141510009765625, | |
| "learning_rate": 3.6124484045416483e-07, | |
| "loss": 0.0001, | |
| "reward": 1.2682291977107525, | |
| "reward_std": 0.24098869320005178, | |
| "rewards/equation_reward_func": 0.2994791720993817, | |
| "rewards/format_reward_func": 0.9687500149011612, | |
| "step": 168 | |
| }, | |
| { | |
| "completion_length": 411.4010524749756, | |
| "epoch": 0.09066666666666667, | |
| "grad_norm": 0.07344633611337249, | |
| "kl": 0.140472412109375, | |
| "learning_rate": 3.580069966606949e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3203125298023224, | |
| "reward_std": 0.22319601802155375, | |
| "rewards/equation_reward_func": 0.3593750062864274, | |
| "rewards/format_reward_func": 0.9609375149011612, | |
| "step": 170 | |
| }, | |
| { | |
| "completion_length": 401.1302185058594, | |
| "epoch": 0.09173333333333333, | |
| "grad_norm": 0.08443145890400267, | |
| "kl": 0.137603759765625, | |
| "learning_rate": 3.547467227712444e-07, | |
| "loss": 0.0001, | |
| "reward": 1.320312537252903, | |
| "reward_std": 0.2728517632931471, | |
| "rewards/equation_reward_func": 0.37500000768341124, | |
| "rewards/format_reward_func": 0.9453125223517418, | |
| "step": 172 | |
| }, | |
| { | |
| "completion_length": 424.09115409851074, | |
| "epoch": 0.0928, | |
| "grad_norm": 0.10948864495158471, | |
| "kl": 0.154022216796875, | |
| "learning_rate": 3.5146469585543386e-07, | |
| "loss": 0.0002, | |
| "reward": 1.302083384245634, | |
| "reward_std": 0.3055522972717881, | |
| "rewards/equation_reward_func": 0.3463541811797768, | |
| "rewards/format_reward_func": 0.9557291902601719, | |
| "step": 174 | |
| }, | |
| { | |
| "completion_length": 419.7265739440918, | |
| "epoch": 0.09386666666666667, | |
| "grad_norm": 0.06846256242159525, | |
| "kl": 0.139007568359375, | |
| "learning_rate": 3.481615975003922e-07, | |
| "loss": 0.0001, | |
| "reward": 1.3333333730697632, | |
| "reward_std": 0.2596399709582329, | |
| "rewards/equation_reward_func": 0.3802083423361182, | |
| "rewards/format_reward_func": 0.9531250149011612, | |
| "step": 176 | |
| }, | |
| { | |
| "completion_length": 422.2161560058594, | |
| "epoch": 0.09493333333333333, | |
| "grad_norm": 0.07246916381967249, | |
| "kl": 0.160308837890625, | |
| "learning_rate": 3.448381136692089e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3125000409781933, | |
| "reward_std": 0.24224651418626308, | |
| "rewards/equation_reward_func": 0.364583347691223, | |
| "rewards/format_reward_func": 0.9479166939854622, | |
| "step": 178 | |
| }, | |
| { | |
| "completion_length": 391.65366077423096, | |
| "epoch": 0.096, | |
| "grad_norm": 0.09628560615962899, | |
| "kl": 0.144439697265625, | |
| "learning_rate": 3.4149493455847897e-07, | |
| "loss": 0.0001, | |
| "reward": 1.460937537252903, | |
| "reward_std": 0.26489327661693096, | |
| "rewards/equation_reward_func": 0.48958334466442466, | |
| "rewards/format_reward_func": 0.9713541753590107, | |
| "step": 180 | |
| }, | |
| { | |
| "completion_length": 393.458345413208, | |
| "epoch": 0.09706666666666666, | |
| "grad_norm": 0.05557570134408156, | |
| "kl": 0.150238037109375, | |
| "learning_rate": 3.3813275445496766e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4036458656191826, | |
| "reward_std": 0.18113364791497588, | |
| "rewards/equation_reward_func": 0.42447917559184134, | |
| "rewards/format_reward_func": 0.9791666828095913, | |
| "step": 182 | |
| }, | |
| { | |
| "completion_length": 430.7395992279053, | |
| "epoch": 0.09813333333333334, | |
| "grad_norm": 0.08433529982073956, | |
| "kl": 0.15069580078125, | |
| "learning_rate": 3.347522715914262e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3828125223517418, | |
| "reward_std": 0.2244554664939642, | |
| "rewards/equation_reward_func": 0.4062500086147338, | |
| "rewards/format_reward_func": 0.9765625149011612, | |
| "step": 184 | |
| }, | |
| { | |
| "completion_length": 375.9218854904175, | |
| "epoch": 0.0992, | |
| "grad_norm": 0.06471574252033455, | |
| "kl": 0.160308837890625, | |
| "learning_rate": 3.313541880015877e-07, | |
| "loss": 0.0002, | |
| "reward": 1.401041716337204, | |
| "reward_std": 0.17062158090993762, | |
| "rewards/equation_reward_func": 0.416666679084301, | |
| "rewards/format_reward_func": 0.9843750111758709, | |
| "step": 186 | |
| }, | |
| { | |
| "completion_length": 386.825532913208, | |
| "epoch": 0.10026666666666667, | |
| "grad_norm": 0.0657902960602302, | |
| "kl": 0.17706298828125, | |
| "learning_rate": 3.279392093743747e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4140625447034836, | |
| "reward_std": 0.1906812135130167, | |
| "rewards/equation_reward_func": 0.4375000155996531, | |
| "rewards/format_reward_func": 0.9765625111758709, | |
| "step": 188 | |
| }, | |
| { | |
| "completion_length": 399.0026168823242, | |
| "epoch": 0.10133333333333333, | |
| "grad_norm": 0.07046713209511725, | |
| "kl": 0.17437744140625, | |
| "learning_rate": 3.245080449073459e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4088541939854622, | |
| "reward_std": 0.22559802932664752, | |
| "rewards/equation_reward_func": 0.4427083448972553, | |
| "rewards/format_reward_func": 0.9661458507180214, | |
| "step": 190 | |
| }, | |
| { | |
| "completion_length": 395.59897232055664, | |
| "epoch": 0.1024, | |
| "grad_norm": 0.04985257435406263, | |
| "kl": 0.30291748046875, | |
| "learning_rate": 3.210614071594162e-07, | |
| "loss": 0.0003, | |
| "reward": 1.3932292014360428, | |
| "reward_std": 0.16054913867264986, | |
| "rewards/equation_reward_func": 0.41406250931322575, | |
| "rewards/format_reward_func": 0.9791666828095913, | |
| "step": 192 | |
| }, | |
| { | |
| "completion_length": 430.26563262939453, | |
| "epoch": 0.10346666666666667, | |
| "grad_norm": 0.0694449694983024, | |
| "kl": 0.157989501953125, | |
| "learning_rate": 3.1760001190287695e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3645833805203438, | |
| "reward_std": 0.22434103582054377, | |
| "rewards/equation_reward_func": 0.390625013737008, | |
| "rewards/format_reward_func": 0.9739583469927311, | |
| "step": 194 | |
| }, | |
| { | |
| "completion_length": 369.6380319595337, | |
| "epoch": 0.10453333333333334, | |
| "grad_norm": 0.062416124903283896, | |
| "kl": 0.17364501953125, | |
| "learning_rate": 3.141245779747502e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4348958730697632, | |
| "reward_std": 0.16678869957104325, | |
| "rewards/equation_reward_func": 0.4505208469927311, | |
| "rewards/format_reward_func": 0.9843750149011612, | |
| "step": 196 | |
| }, | |
| { | |
| "completion_length": 375.2916774749756, | |
| "epoch": 0.1056, | |
| "grad_norm": 0.06792569091679665, | |
| "kl": 0.1728515625, | |
| "learning_rate": 3.106358271275056e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4062500409781933, | |
| "reward_std": 0.2316625458188355, | |
| "rewards/equation_reward_func": 0.45312500931322575, | |
| "rewards/format_reward_func": 0.9531250223517418, | |
| "step": 198 | |
| }, | |
| { | |
| "completion_length": 353.49219608306885, | |
| "epoch": 0.10666666666666667, | |
| "grad_norm": 0.05513526342996937, | |
| "kl": 0.21026611328125, | |
| "learning_rate": 3.0713448387917227e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4635417088866234, | |
| "reward_std": 0.15594629338011146, | |
| "rewards/equation_reward_func": 0.48177084559574723, | |
| "rewards/format_reward_func": 0.9817708469927311, | |
| "step": 200 | |
| }, | |
| { | |
| "completion_length": 406.79688453674316, | |
| "epoch": 0.10773333333333333, | |
| "grad_norm": 0.08708737710543844, | |
| "kl": 0.373809814453125, | |
| "learning_rate": 3.0362127536287636e-07, | |
| "loss": 0.0004, | |
| "reward": 1.3359375335276127, | |
| "reward_std": 0.190874763764441, | |
| "rewards/equation_reward_func": 0.3645833421032876, | |
| "rewards/format_reward_func": 0.9713541828095913, | |
| "step": 202 | |
| }, | |
| { | |
| "completion_length": 388.12501335144043, | |
| "epoch": 0.1088, | |
| "grad_norm": 0.06755151405749991, | |
| "kl": 0.190673828125, | |
| "learning_rate": 3.0009693117583523e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3255208693444729, | |
| "reward_std": 0.20536378538236022, | |
| "rewards/equation_reward_func": 0.3697916781529784, | |
| "rewards/format_reward_func": 0.9557291902601719, | |
| "step": 204 | |
| }, | |
| { | |
| "completion_length": 341.47917461395264, | |
| "epoch": 0.10986666666666667, | |
| "grad_norm": 0.07143070261021141, | |
| "kl": 0.1871337890625, | |
| "learning_rate": 2.965621832278401e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4609375558793545, | |
| "reward_std": 0.16247200826182961, | |
| "rewards/equation_reward_func": 0.48437500977888703, | |
| "rewards/format_reward_func": 0.9765625149011612, | |
| "step": 206 | |
| }, | |
| { | |
| "completion_length": 367.3541793823242, | |
| "epoch": 0.11093333333333333, | |
| "grad_norm": 0.05250515001971676, | |
| "kl": 0.186767578125, | |
| "learning_rate": 2.9301776558925875e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3802083656191826, | |
| "reward_std": 0.16997373616322875, | |
| "rewards/equation_reward_func": 0.40104167722165585, | |
| "rewards/format_reward_func": 0.979166679084301, | |
| "step": 208 | |
| }, | |
| { | |
| "completion_length": 323.38021755218506, | |
| "epoch": 0.112, | |
| "grad_norm": 0.05958909368430716, | |
| "kl": 0.20263671875, | |
| "learning_rate": 2.894644143385885e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4869792014360428, | |
| "reward_std": 0.15834738174453378, | |
| "rewards/equation_reward_func": 0.5078125111758709, | |
| "rewards/format_reward_func": 0.979166679084301, | |
| "step": 210 | |
| }, | |
| { | |
| "completion_length": 352.716157913208, | |
| "epoch": 0.11306666666666666, | |
| "grad_norm": 0.052786746286241845, | |
| "kl": 0.1922607421875, | |
| "learning_rate": 2.859028674095937e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4401042088866234, | |
| "reward_std": 0.15080112405121326, | |
| "rewards/equation_reward_func": 0.4453125100117177, | |
| "rewards/format_reward_func": 0.9947916716337204, | |
| "step": 212 | |
| }, | |
| { | |
| "completion_length": 344.7161569595337, | |
| "epoch": 0.11413333333333334, | |
| "grad_norm": 0.08811072664401991, | |
| "kl": 0.19512939453125, | |
| "learning_rate": 2.823338644380566e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4296875484287739, | |
| "reward_std": 0.13642758363857865, | |
| "rewards/equation_reward_func": 0.44010418094694614, | |
| "rewards/format_reward_func": 0.9895833395421505, | |
| "step": 214 | |
| }, | |
| { | |
| "completion_length": 366.8854274749756, | |
| "epoch": 0.1152, | |
| "grad_norm": 0.0667929740425298, | |
| "kl": 0.19061279296875, | |
| "learning_rate": 2.7875814660817504e-07, | |
| "loss": 0.0002, | |
| "reward": 1.385416705161333, | |
| "reward_std": 0.24201409425586462, | |
| "rewards/equation_reward_func": 0.4114583421032876, | |
| "rewards/format_reward_func": 0.9739583469927311, | |
| "step": 216 | |
| }, | |
| { | |
| "completion_length": 331.97396755218506, | |
| "epoch": 0.11626666666666667, | |
| "grad_norm": 0.06176909332442502, | |
| "kl": 0.2694091796875, | |
| "learning_rate": 2.751764564986396e-07, | |
| "loss": 0.0003, | |
| "reward": 1.4817708805203438, | |
| "reward_std": 0.1628231555223465, | |
| "rewards/equation_reward_func": 0.5026041772216558, | |
| "rewards/format_reward_func": 0.979166679084301, | |
| "step": 218 | |
| }, | |
| { | |
| "completion_length": 372.8307399749756, | |
| "epoch": 0.11733333333333333, | |
| "grad_norm": 0.061937013664825874, | |
| "kl": 0.1832275390625, | |
| "learning_rate": 2.715895379284194e-07, | |
| "loss": 0.0002, | |
| "reward": 1.346354216337204, | |
| "reward_std": 0.16828062618151307, | |
| "rewards/equation_reward_func": 0.3671875095460564, | |
| "rewards/format_reward_func": 0.9791666828095913, | |
| "step": 220 | |
| }, | |
| { | |
| "completion_length": 349.80469608306885, | |
| "epoch": 0.1184, | |
| "grad_norm": 0.05527110509029042, | |
| "kl": 0.20123291015625, | |
| "learning_rate": 2.6799813580229174e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3984375521540642, | |
| "reward_std": 0.15229387395083904, | |
| "rewards/equation_reward_func": 0.4114583465270698, | |
| "rewards/format_reward_func": 0.9869791753590107, | |
| "step": 222 | |
| }, | |
| { | |
| "completion_length": 389.661470413208, | |
| "epoch": 0.11946666666666667, | |
| "grad_norm": 0.034929544543460735, | |
| "kl": 0.18328857421875, | |
| "learning_rate": 2.6440299595614606e-07, | |
| "loss": 0.0002, | |
| "reward": 1.2942708916962147, | |
| "reward_std": 0.12109041679650545, | |
| "rewards/equation_reward_func": 0.32031251094304025, | |
| "rewards/format_reward_func": 0.9739583507180214, | |
| "step": 224 | |
| }, | |
| { | |
| "completion_length": 304.37761211395264, | |
| "epoch": 0.12053333333333334, | |
| "grad_norm": 1.959456454356156, | |
| "kl": 0.1982421875, | |
| "learning_rate": 2.6080486500209347e-07, | |
| "loss": 0.0002, | |
| "reward": 1.5494792014360428, | |
| "reward_std": 0.16492673894390464, | |
| "rewards/equation_reward_func": 0.5651041818782687, | |
| "rewards/format_reward_func": 0.9843750149011612, | |
| "step": 226 | |
| }, | |
| { | |
| "completion_length": 327.7291774749756, | |
| "epoch": 0.1216, | |
| "grad_norm": 0.05873873809239406, | |
| "kl": 0.234130859375, | |
| "learning_rate": 2.572044901734166e-07, | |
| "loss": 0.0002, | |
| "reward": 1.5130208767950535, | |
| "reward_std": 0.2238043532706797, | |
| "rewards/equation_reward_func": 0.5416666837409139, | |
| "rewards/format_reward_func": 0.971354179084301, | |
| "step": 228 | |
| }, | |
| { | |
| "completion_length": 352.1250114440918, | |
| "epoch": 0.12266666666666666, | |
| "grad_norm": 0.053179730956003986, | |
| "kl": 0.201171875, | |
| "learning_rate": 2.536026191693893e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3828125447034836, | |
| "reward_std": 0.170977798756212, | |
| "rewards/equation_reward_func": 0.4062500090803951, | |
| "rewards/format_reward_func": 0.9765625149011612, | |
| "step": 230 | |
| }, | |
| { | |
| "completion_length": 318.364595413208, | |
| "epoch": 0.12373333333333333, | |
| "grad_norm": 0.09137226114418516, | |
| "kl": 0.2119140625, | |
| "learning_rate": 2.5e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4895833656191826, | |
| "reward_std": 0.19272029865533113, | |
| "rewards/equation_reward_func": 0.5156250144354999, | |
| "rewards/format_reward_func": 0.9739583469927311, | |
| "step": 232 | |
| }, | |
| { | |
| "completion_length": 338.41667556762695, | |
| "epoch": 0.1248, | |
| "grad_norm": 0.03473787219752575, | |
| "kl": 0.2049560546875, | |
| "learning_rate": 2.4639738083061073e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4427083805203438, | |
| "reward_std": 0.1410418525338173, | |
| "rewards/equation_reward_func": 0.45833334885537624, | |
| "rewards/format_reward_func": 0.9843750074505806, | |
| "step": 234 | |
| }, | |
| { | |
| "completion_length": 328.80469512939453, | |
| "epoch": 0.12586666666666665, | |
| "grad_norm": 0.05170053991702217, | |
| "kl": 0.277099609375, | |
| "learning_rate": 2.4279550982658345e-07, | |
| "loss": 0.0003, | |
| "reward": 1.4244791977107525, | |
| "reward_std": 0.12529170233756304, | |
| "rewards/equation_reward_func": 0.44531250768341124, | |
| "rewards/format_reward_func": 0.979166679084301, | |
| "step": 236 | |
| }, | |
| { | |
| "completion_length": 305.9505310058594, | |
| "epoch": 0.12693333333333334, | |
| "grad_norm": 0.07000303665038683, | |
| "kl": 0.2147216796875, | |
| "learning_rate": 2.3919513499790646e-07, | |
| "loss": 0.0002, | |
| "reward": 1.5390625223517418, | |
| "reward_std": 0.19656797824427485, | |
| "rewards/equation_reward_func": 0.5494791772216558, | |
| "rewards/format_reward_func": 0.9895833432674408, | |
| "step": 238 | |
| }, | |
| { | |
| "completion_length": 327.80209255218506, | |
| "epoch": 0.128, | |
| "grad_norm": 0.052615354107808474, | |
| "kl": 0.2071533203125, | |
| "learning_rate": 2.3559700404385394e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4661458767950535, | |
| "reward_std": 0.19014028925448656, | |
| "rewards/equation_reward_func": 0.4843750144354999, | |
| "rewards/format_reward_func": 0.9817708469927311, | |
| "step": 240 | |
| }, | |
| { | |
| "completion_length": 339.92709159851074, | |
| "epoch": 0.12906666666666666, | |
| "grad_norm": 0.06447582251906563, | |
| "kl": 0.2037353515625, | |
| "learning_rate": 2.3200186419770823e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3958333767950535, | |
| "reward_std": 0.2038758508861065, | |
| "rewards/equation_reward_func": 0.4244791760575026, | |
| "rewards/format_reward_func": 0.9713541828095913, | |
| "step": 242 | |
| }, | |
| { | |
| "completion_length": 344.87761878967285, | |
| "epoch": 0.13013333333333332, | |
| "grad_norm": 0.058764936796754055, | |
| "kl": 0.20806884765625, | |
| "learning_rate": 2.284104620715807e-07, | |
| "loss": 0.0002, | |
| "reward": 1.442708395421505, | |
| "reward_std": 0.13642922416329384, | |
| "rewards/equation_reward_func": 0.4557291795499623, | |
| "rewards/format_reward_func": 0.986979179084301, | |
| "step": 244 | |
| }, | |
| { | |
| "completion_length": 303.09375858306885, | |
| "epoch": 0.1312, | |
| "grad_norm": 0.037716918562136104, | |
| "kl": 0.22711181640625, | |
| "learning_rate": 2.2482354350136043e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4713542014360428, | |
| "reward_std": 0.10845672665163875, | |
| "rewards/equation_reward_func": 0.4817708421032876, | |
| "rewards/format_reward_func": 0.9895833432674408, | |
| "step": 246 | |
| }, | |
| { | |
| "completion_length": 330.54688358306885, | |
| "epoch": 0.13226666666666667, | |
| "grad_norm": 0.05875241729951767, | |
| "kl": 0.23126220703125, | |
| "learning_rate": 2.2124185339182496e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4322917088866234, | |
| "reward_std": 0.2153621120378375, | |
| "rewards/equation_reward_func": 0.4765625149011612, | |
| "rewards/format_reward_func": 0.9557291865348816, | |
| "step": 248 | |
| }, | |
| { | |
| "completion_length": 327.26303005218506, | |
| "epoch": 0.13333333333333333, | |
| "grad_norm": 0.059797262492205065, | |
| "kl": 0.21978759765625, | |
| "learning_rate": 2.1766613556194344e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4218750335276127, | |
| "reward_std": 0.15325831901282072, | |
| "rewards/equation_reward_func": 0.44270834513008595, | |
| "rewards/format_reward_func": 0.9791666753590107, | |
| "step": 250 | |
| }, | |
| { | |
| "completion_length": 331.7838649749756, | |
| "epoch": 0.1344, | |
| "grad_norm": 0.05584852018689532, | |
| "kl": 0.22015380859375, | |
| "learning_rate": 2.1409713259040628e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4192708693444729, | |
| "reward_std": 0.16641948884353042, | |
| "rewards/equation_reward_func": 0.4505208460614085, | |
| "rewards/format_reward_func": 0.9687500186264515, | |
| "step": 252 | |
| }, | |
| { | |
| "completion_length": 342.6458396911621, | |
| "epoch": 0.13546666666666668, | |
| "grad_norm": 0.05324434721305267, | |
| "kl": 0.20458984375, | |
| "learning_rate": 2.105355856614115e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4010417126119137, | |
| "reward_std": 0.16240685293450952, | |
| "rewards/equation_reward_func": 0.42447918094694614, | |
| "rewards/format_reward_func": 0.9765625111758709, | |
| "step": 254 | |
| }, | |
| { | |
| "completion_length": 319.91407203674316, | |
| "epoch": 0.13653333333333334, | |
| "grad_norm": 0.07130764085756268, | |
| "kl": 0.21002197265625, | |
| "learning_rate": 2.069822344107413e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4505208730697632, | |
| "reward_std": 0.13851393247023225, | |
| "rewards/equation_reward_func": 0.4661458432674408, | |
| "rewards/format_reward_func": 0.9843750111758709, | |
| "step": 256 | |
| }, | |
| { | |
| "completion_length": 319.0599031448364, | |
| "epoch": 0.1376, | |
| "grad_norm": 0.05793306781687775, | |
| "kl": 0.23541259765625, | |
| "learning_rate": 2.034378167721599e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4661458805203438, | |
| "reward_std": 0.16103536309674382, | |
| "rewards/equation_reward_func": 0.4843750153668225, | |
| "rewards/format_reward_func": 0.9817708469927311, | |
| "step": 258 | |
| }, | |
| { | |
| "completion_length": 298.2187557220459, | |
| "epoch": 0.13866666666666666, | |
| "grad_norm": 0.04829645671147647, | |
| "kl": 0.2176513671875, | |
| "learning_rate": 1.9990306882416485e-07, | |
| "loss": 0.0002, | |
| "reward": 1.5104167088866234, | |
| "reward_std": 0.10582649055868387, | |
| "rewards/equation_reward_func": 0.5208333458285779, | |
| "rewards/format_reward_func": 0.9895833432674408, | |
| "step": 260 | |
| }, | |
| { | |
| "completion_length": 317.6067781448364, | |
| "epoch": 0.13973333333333332, | |
| "grad_norm": 0.07057516185361917, | |
| "kl": 0.267578125, | |
| "learning_rate": 1.9637872463712362e-07, | |
| "loss": 0.0003, | |
| "reward": 1.4557292088866234, | |
| "reward_std": 0.16413959069177508, | |
| "rewards/equation_reward_func": 0.4765625100117177, | |
| "rewards/format_reward_func": 0.979166679084301, | |
| "step": 262 | |
| }, | |
| { | |
| "completion_length": 315.09375762939453, | |
| "epoch": 0.1408, | |
| "grad_norm": 0.03516233269281731, | |
| "kl": 0.21759033203125, | |
| "learning_rate": 1.9286551612082773e-07, | |
| "loss": 0.0002, | |
| "reward": 1.5104167088866234, | |
| "reward_std": 0.1660658805631101, | |
| "rewards/equation_reward_func": 0.5286458479240537, | |
| "rewards/format_reward_func": 0.9817708432674408, | |
| "step": 264 | |
| }, | |
| { | |
| "completion_length": 305.10417461395264, | |
| "epoch": 0.14186666666666667, | |
| "grad_norm": 0.06161456176400378, | |
| "kl": 0.26385498046875, | |
| "learning_rate": 1.8936417287249446e-07, | |
| "loss": 0.0003, | |
| "reward": 1.4687500447034836, | |
| "reward_std": 0.15946846222504973, | |
| "rewards/equation_reward_func": 0.5000000121071935, | |
| "rewards/format_reward_func": 0.9687500186264515, | |
| "step": 266 | |
| }, | |
| { | |
| "completion_length": 296.25782012939453, | |
| "epoch": 0.14293333333333333, | |
| "grad_norm": 0.03532637058161108, | |
| "kl": 0.24688720703125, | |
| "learning_rate": 1.8587542202524985e-07, | |
| "loss": 0.0002, | |
| "reward": 1.5000000596046448, | |
| "reward_std": 0.09775482444092631, | |
| "rewards/equation_reward_func": 0.5156250167638063, | |
| "rewards/format_reward_func": 0.9843750074505806, | |
| "step": 268 | |
| }, | |
| { | |
| "completion_length": 319.60417461395264, | |
| "epoch": 0.144, | |
| "grad_norm": 0.05825621213686169, | |
| "kl": 0.22161865234375, | |
| "learning_rate": 1.82399988097123e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4609375298023224, | |
| "reward_std": 0.13511197874322534, | |
| "rewards/equation_reward_func": 0.4739583453629166, | |
| "rewards/format_reward_func": 0.9869791716337204, | |
| "step": 270 | |
| }, | |
| { | |
| "completion_length": 304.9088673591614, | |
| "epoch": 0.14506666666666668, | |
| "grad_norm": 0.07890476653259258, | |
| "kl": 0.23291015625, | |
| "learning_rate": 1.7893859284058378e-07, | |
| "loss": 0.0002, | |
| "reward": 1.479166705161333, | |
| "reward_std": 0.17851981101557612, | |
| "rewards/equation_reward_func": 0.5026041809469461, | |
| "rewards/format_reward_func": 0.9765625111758709, | |
| "step": 272 | |
| }, | |
| { | |
| "completion_length": 311.20573711395264, | |
| "epoch": 0.14613333333333334, | |
| "grad_norm": 0.0696534900496278, | |
| "kl": 0.2391357421875, | |
| "learning_rate": 1.7549195509265407e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4921875521540642, | |
| "reward_std": 0.2119438904337585, | |
| "rewards/equation_reward_func": 0.5182291865348816, | |
| "rewards/format_reward_func": 0.9739583507180214, | |
| "step": 274 | |
| }, | |
| { | |
| "completion_length": 328.24219512939453, | |
| "epoch": 0.1472, | |
| "grad_norm": 0.06610853694110924, | |
| "kl": 0.2369384765625, | |
| "learning_rate": 1.7206079062562536e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4192708656191826, | |
| "reward_std": 0.18404160952195525, | |
| "rewards/equation_reward_func": 0.4661458460614085, | |
| "rewards/format_reward_func": 0.9531250149011612, | |
| "step": 276 | |
| }, | |
| { | |
| "completion_length": 340.9791717529297, | |
| "epoch": 0.14826666666666666, | |
| "grad_norm": 0.07464727691456224, | |
| "kl": 0.24871826171875, | |
| "learning_rate": 1.6864581199841226e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3958333693444729, | |
| "reward_std": 0.17743596993386745, | |
| "rewards/equation_reward_func": 0.43750000931322575, | |
| "rewards/format_reward_func": 0.9583333544433117, | |
| "step": 278 | |
| }, | |
| { | |
| "completion_length": 310.48698806762695, | |
| "epoch": 0.14933333333333335, | |
| "grad_norm": 0.06469138106157636, | |
| "kl": 0.23760986328125, | |
| "learning_rate": 1.6524772840857388e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4973958656191826, | |
| "reward_std": 0.23611926287412643, | |
| "rewards/equation_reward_func": 0.526041672565043, | |
| "rewards/format_reward_func": 0.9713541828095913, | |
| "step": 280 | |
| }, | |
| { | |
| "completion_length": 354.2942810058594, | |
| "epoch": 0.1504, | |
| "grad_norm": 0.07410196368141982, | |
| "kl": 0.2314453125, | |
| "learning_rate": 1.6186724554503237e-07, | |
| "loss": 0.0002, | |
| "reward": 1.354166705161333, | |
| "reward_std": 0.2034895895048976, | |
| "rewards/equation_reward_func": 0.39322918094694614, | |
| "rewards/format_reward_func": 0.9609375186264515, | |
| "step": 282 | |
| }, | |
| { | |
| "completion_length": 381.19011402130127, | |
| "epoch": 0.15146666666666667, | |
| "grad_norm": 0.0594094806781079, | |
| "kl": 0.21417236328125, | |
| "learning_rate": 1.5850506544152103e-07, | |
| "loss": 0.0002, | |
| "reward": 1.2578125298023224, | |
| "reward_std": 0.22114716423675418, | |
| "rewards/equation_reward_func": 0.3125000074505806, | |
| "rewards/format_reward_func": 0.9453125223517418, | |
| "step": 284 | |
| }, | |
| { | |
| "completion_length": 287.7083406448364, | |
| "epoch": 0.15253333333333333, | |
| "grad_norm": 0.048529837958803314, | |
| "kl": 0.2618408203125, | |
| "learning_rate": 1.5516188633079107e-07, | |
| "loss": 0.0003, | |
| "reward": 1.5494792088866234, | |
| "reward_std": 0.14577306527644396, | |
| "rewards/equation_reward_func": 0.5729166772216558, | |
| "rewards/format_reward_func": 0.9765625111758709, | |
| "step": 286 | |
| }, | |
| { | |
| "completion_length": 349.13541984558105, | |
| "epoch": 0.1536, | |
| "grad_norm": 0.0547733516033155, | |
| "kl": 0.27655029296875, | |
| "learning_rate": 1.5183840249960784e-07, | |
| "loss": 0.0003, | |
| "reward": 1.3802083805203438, | |
| "reward_std": 0.1545771942473948, | |
| "rewards/equation_reward_func": 0.4296875111758709, | |
| "rewards/format_reward_func": 0.9505208544433117, | |
| "step": 288 | |
| }, | |
| { | |
| "completion_length": 330.4791736602783, | |
| "epoch": 0.15466666666666667, | |
| "grad_norm": 0.07157642441413668, | |
| "kl": 0.232421875, | |
| "learning_rate": 1.4853530414456612e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4505208916962147, | |
| "reward_std": 0.17313488014042377, | |
| "rewards/equation_reward_func": 0.484375013737008, | |
| "rewards/format_reward_func": 0.9661458507180214, | |
| "step": 290 | |
| }, | |
| { | |
| "completion_length": 349.14844703674316, | |
| "epoch": 0.15573333333333333, | |
| "grad_norm": 0.07116589129452708, | |
| "kl": 0.271240234375, | |
| "learning_rate": 1.4525327722875568e-07, | |
| "loss": 0.0003, | |
| "reward": 1.4088542088866234, | |
| "reward_std": 0.1796160596422851, | |
| "rewards/equation_reward_func": 0.4531250111758709, | |
| "rewards/format_reward_func": 0.9557291828095913, | |
| "step": 292 | |
| }, | |
| { | |
| "completion_length": 319.7448024749756, | |
| "epoch": 0.1568, | |
| "grad_norm": 0.06230251147212293, | |
| "kl": 0.249267578125, | |
| "learning_rate": 1.4199300333930515e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4531250521540642, | |
| "reward_std": 0.20882080029696226, | |
| "rewards/equation_reward_func": 0.49479168094694614, | |
| "rewards/format_reward_func": 0.9583333544433117, | |
| "step": 294 | |
| }, | |
| { | |
| "completion_length": 333.73698711395264, | |
| "epoch": 0.15786666666666666, | |
| "grad_norm": 0.06272569248842443, | |
| "kl": 0.2855224609375, | |
| "learning_rate": 1.3875515954583523e-07, | |
| "loss": 0.0003, | |
| "reward": 1.4739583656191826, | |
| "reward_std": 0.17252166103571653, | |
| "rewards/equation_reward_func": 0.513020845130086, | |
| "rewards/format_reward_func": 0.9609375223517418, | |
| "step": 296 | |
| }, | |
| { | |
| "completion_length": 325.72396421432495, | |
| "epoch": 0.15893333333333334, | |
| "grad_norm": 0.06294341366186487, | |
| "kl": 0.25537109375, | |
| "learning_rate": 1.3554041825985e-07, | |
| "loss": 0.0003, | |
| "reward": 1.4479167014360428, | |
| "reward_std": 0.16139157954603434, | |
| "rewards/equation_reward_func": 0.4869791797827929, | |
| "rewards/format_reward_func": 0.9609375186264515, | |
| "step": 298 | |
| }, | |
| { | |
| "completion_length": 346.55990505218506, | |
| "epoch": 0.16, | |
| "grad_norm": 0.08834891258610764, | |
| "kl": 0.225341796875, | |
| "learning_rate": 1.323494470950949e-07, | |
| "loss": 0.0002, | |
| "reward": 1.421875037252903, | |
| "reward_std": 0.21047524316236377, | |
| "rewards/equation_reward_func": 0.4609375102445483, | |
| "rewards/format_reward_func": 0.9609375186264515, | |
| "step": 300 | |
| }, | |
| { | |
| "completion_length": 352.68230056762695, | |
| "epoch": 0.16106666666666666, | |
| "grad_norm": 0.057803994549006414, | |
| "kl": 0.22314453125, | |
| "learning_rate": 1.2918290872891236e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3671875484287739, | |
| "reward_std": 0.14487217646092176, | |
| "rewards/equation_reward_func": 0.41927084513008595, | |
| "rewards/format_reward_func": 0.9479166828095913, | |
| "step": 302 | |
| }, | |
| { | |
| "completion_length": 333.1145920753479, | |
| "epoch": 0.16213333333333332, | |
| "grad_norm": 0.058783183649954864, | |
| "kl": 0.24957275390625, | |
| "learning_rate": 1.260414607646213e-07, | |
| "loss": 0.0002, | |
| "reward": 1.453125037252903, | |
| "reward_std": 0.16050026565790176, | |
| "rewards/equation_reward_func": 0.49739584350027144, | |
| "rewards/format_reward_func": 0.9557291828095913, | |
| "step": 304 | |
| }, | |
| { | |
| "completion_length": 341.60157012939453, | |
| "epoch": 0.1632, | |
| "grad_norm": 0.07522161581548838, | |
| "kl": 0.22479248046875, | |
| "learning_rate": 1.2292575559495143e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4062500447034836, | |
| "reward_std": 0.22941595874726772, | |
| "rewards/equation_reward_func": 0.46354168467223644, | |
| "rewards/format_reward_func": 0.9427083544433117, | |
| "step": 306 | |
| }, | |
| { | |
| "completion_length": 338.2005338668823, | |
| "epoch": 0.16426666666666667, | |
| "grad_norm": 0.06752892615049455, | |
| "kl": 0.23663330078125, | |
| "learning_rate": 1.1983644026655835e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4453125521540642, | |
| "reward_std": 0.2164831915870309, | |
| "rewards/equation_reward_func": 0.5026041837409139, | |
| "rewards/format_reward_func": 0.9427083507180214, | |
| "step": 308 | |
| }, | |
| { | |
| "completion_length": 361.3724021911621, | |
| "epoch": 0.16533333333333333, | |
| "grad_norm": 0.06676421692912096, | |
| "kl": 0.22576904296875, | |
| "learning_rate": 1.1677415634565066e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3489583730697632, | |
| "reward_std": 0.20666454918682575, | |
| "rewards/equation_reward_func": 0.40104167768731713, | |
| "rewards/format_reward_func": 0.9479166939854622, | |
| "step": 310 | |
| }, | |
| { | |
| "completion_length": 357.0937604904175, | |
| "epoch": 0.1664, | |
| "grad_norm": 0.04983945410467141, | |
| "kl": 0.26348876953125, | |
| "learning_rate": 1.1373953978475353e-07, | |
| "loss": 0.0003, | |
| "reward": 1.393229216337204, | |
| "reward_std": 0.23145765578374267, | |
| "rewards/equation_reward_func": 0.4739583469927311, | |
| "rewards/format_reward_func": 0.9192708469927311, | |
| "step": 312 | |
| }, | |
| { | |
| "completion_length": 334.27344512939453, | |
| "epoch": 0.16746666666666668, | |
| "grad_norm": 0.0733748328273072, | |
| "kl": 0.2396240234375, | |
| "learning_rate": 1.1073322079063913e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4661458730697632, | |
| "reward_std": 0.23468726128339767, | |
| "rewards/equation_reward_func": 0.5312500186264515, | |
| "rewards/format_reward_func": 0.9348958618938923, | |
| "step": 314 | |
| }, | |
| { | |
| "completion_length": 355.0312604904175, | |
| "epoch": 0.16853333333333334, | |
| "grad_norm": 0.08049805206787042, | |
| "kl": 0.2210693359375, | |
| "learning_rate": 1.0775582369344946e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4244791977107525, | |
| "reward_std": 0.22463522851467133, | |
| "rewards/equation_reward_func": 0.46875001094304025, | |
| "rewards/format_reward_func": 0.955729179084301, | |
| "step": 316 | |
| }, | |
| { | |
| "completion_length": 364.07813930511475, | |
| "epoch": 0.1696, | |
| "grad_norm": 0.06940364182839838, | |
| "kl": 0.25628662109375, | |
| "learning_rate": 1.0480796681704077e-07, | |
| "loss": 0.0003, | |
| "reward": 1.3515625298023224, | |
| "reward_std": 0.25020841229707, | |
| "rewards/equation_reward_func": 0.4114583458285779, | |
| "rewards/format_reward_func": 0.9401041939854622, | |
| "step": 318 | |
| }, | |
| { | |
| "completion_length": 361.2630319595337, | |
| "epoch": 0.17066666666666666, | |
| "grad_norm": 0.05999980167906364, | |
| "kl": 0.2413330078125, | |
| "learning_rate": 1.018902623505741e-07, | |
| "loss": 0.0002, | |
| "reward": 1.3411458767950535, | |
| "reward_std": 0.1776549629867077, | |
| "rewards/equation_reward_func": 0.40104167722165585, | |
| "rewards/format_reward_func": 0.9401041865348816, | |
| "step": 320 | |
| }, | |
| { | |
| "completion_length": 373.94271659851074, | |
| "epoch": 0.17173333333333332, | |
| "grad_norm": 0.06439424679796442, | |
| "kl": 0.25555419921875, | |
| "learning_rate": 9.900331622138063e-08, | |
| "loss": 0.0003, | |
| "reward": 1.3307292126119137, | |
| "reward_std": 0.23397412337362766, | |
| "rewards/equation_reward_func": 0.398437513737008, | |
| "rewards/format_reward_func": 0.932291679084301, | |
| "step": 322 | |
| }, | |
| { | |
| "completion_length": 394.7474069595337, | |
| "epoch": 0.1728, | |
| "grad_norm": 0.05964680052659955, | |
| "kl": 0.21405029296875, | |
| "learning_rate": 9.614772796912681e-08, | |
| "loss": 0.0002, | |
| "reward": 1.3151041939854622, | |
| "reward_std": 0.24081564601510763, | |
| "rewards/equation_reward_func": 0.37239584047347307, | |
| "rewards/format_reward_func": 0.9427083507180214, | |
| "step": 324 | |
| }, | |
| { | |
| "completion_length": 334.4062604904175, | |
| "epoch": 0.17386666666666667, | |
| "grad_norm": 0.06010536886525251, | |
| "kl": 0.24920654296875, | |
| "learning_rate": 9.332409062130686e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4895833656191826, | |
| "reward_std": 0.23100403556600213, | |
| "rewards/equation_reward_func": 0.5468750167638063, | |
| "rewards/format_reward_func": 0.9427083544433117, | |
| "step": 326 | |
| }, | |
| { | |
| "completion_length": 339.3489656448364, | |
| "epoch": 0.17493333333333333, | |
| "grad_norm": 0.058812330078301236, | |
| "kl": 0.23675537109375, | |
| "learning_rate": 9.053299057008699e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4427083656191826, | |
| "reward_std": 0.22872037440538406, | |
| "rewards/equation_reward_func": 0.5000000174622983, | |
| "rewards/format_reward_func": 0.942708358168602, | |
| "step": 328 | |
| }, | |
| { | |
| "completion_length": 358.95573711395264, | |
| "epoch": 0.176, | |
| "grad_norm": 0.047621349241250256, | |
| "kl": 0.3682861328125, | |
| "learning_rate": 8.777500745052743e-08, | |
| "loss": 0.0004, | |
| "reward": 1.4140625409781933, | |
| "reward_std": 0.2034787591546774, | |
| "rewards/equation_reward_func": 0.468750013737008, | |
| "rewards/format_reward_func": 0.9453125186264515, | |
| "step": 330 | |
| }, | |
| { | |
| "completion_length": 341.7994909286499, | |
| "epoch": 0.17706666666666668, | |
| "grad_norm": 0.056517627432135925, | |
| "kl": 0.22198486328125, | |
| "learning_rate": 8.505071402020892e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4661458730697632, | |
| "reward_std": 0.24763745069503784, | |
| "rewards/equation_reward_func": 0.5130208497866988, | |
| "rewards/format_reward_func": 0.9531250186264515, | |
| "step": 332 | |
| }, | |
| { | |
| "completion_length": 304.9583435058594, | |
| "epoch": 0.17813333333333334, | |
| "grad_norm": 0.050262969260993634, | |
| "kl": 0.2586669921875, | |
| "learning_rate": 8.236067604028562e-08, | |
| "loss": 0.0003, | |
| "reward": 1.565104205161333, | |
| "reward_std": 0.18649716302752495, | |
| "rewards/equation_reward_func": 0.5989583495538682, | |
| "rewards/format_reward_func": 0.9661458507180214, | |
| "step": 334 | |
| }, | |
| { | |
| "completion_length": 342.1041736602783, | |
| "epoch": 0.1792, | |
| "grad_norm": 0.05716863952766954, | |
| "kl": 0.22137451171875, | |
| "learning_rate": 7.970545215799327e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4739583805203438, | |
| "reward_std": 0.21373012708500028, | |
| "rewards/equation_reward_func": 0.5234375135041773, | |
| "rewards/format_reward_func": 0.9505208544433117, | |
| "step": 336 | |
| }, | |
| { | |
| "completion_length": 308.6770896911621, | |
| "epoch": 0.18026666666666666, | |
| "grad_norm": 0.0733889706843634, | |
| "kl": 0.2593994140625, | |
| "learning_rate": 7.708559379063204e-08, | |
| "loss": 0.0003, | |
| "reward": 1.492187537252903, | |
| "reward_std": 0.17253324948251247, | |
| "rewards/equation_reward_func": 0.523437513038516, | |
| "rewards/format_reward_func": 0.9687500111758709, | |
| "step": 338 | |
| }, | |
| { | |
| "completion_length": 399.8151168823242, | |
| "epoch": 0.18133333333333335, | |
| "grad_norm": 0.05908917640237805, | |
| "kl": 0.21478271484375, | |
| "learning_rate": 7.45016450110534e-08, | |
| "loss": 0.0002, | |
| "reward": 1.3515625447034836, | |
| "reward_std": 0.26676404289901257, | |
| "rewards/equation_reward_func": 0.424479172565043, | |
| "rewards/format_reward_func": 0.9270833507180214, | |
| "step": 340 | |
| }, | |
| { | |
| "completion_length": 366.43750858306885, | |
| "epoch": 0.1824, | |
| "grad_norm": 0.06086281604379656, | |
| "kl": 0.2493896484375, | |
| "learning_rate": 7.195414243467029e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4062500298023224, | |
| "reward_std": 0.21875010011717677, | |
| "rewards/equation_reward_func": 0.4583333395421505, | |
| "rewards/format_reward_func": 0.9479166828095913, | |
| "step": 342 | |
| }, | |
| { | |
| "completion_length": 334.81250953674316, | |
| "epoch": 0.18346666666666667, | |
| "grad_norm": 0.07410531250671612, | |
| "kl": 0.22772216796875, | |
| "learning_rate": 6.944361510801763e-08, | |
| "loss": 0.0002, | |
| "reward": 1.471354205161333, | |
| "reward_std": 0.30193538405001163, | |
| "rewards/equation_reward_func": 0.5312500174622983, | |
| "rewards/format_reward_func": 0.9401041902601719, | |
| "step": 344 | |
| }, | |
| { | |
| "completion_length": 393.6250114440918, | |
| "epoch": 0.18453333333333333, | |
| "grad_norm": 0.06337236320877422, | |
| "kl": 0.2236328125, | |
| "learning_rate": 6.697058439888283e-08, | |
| "loss": 0.0002, | |
| "reward": 1.2890625298023224, | |
| "reward_std": 0.2558422018773854, | |
| "rewards/equation_reward_func": 0.36197917559184134, | |
| "rewards/format_reward_func": 0.9270833544433117, | |
| "step": 346 | |
| }, | |
| { | |
| "completion_length": 344.8932418823242, | |
| "epoch": 0.1856, | |
| "grad_norm": 0.07154870009295032, | |
| "kl": 0.23614501953125, | |
| "learning_rate": 6.453556388803288e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4140625223517418, | |
| "reward_std": 0.24450676841661334, | |
| "rewards/equation_reward_func": 0.4817708474583924, | |
| "rewards/format_reward_func": 0.9322916902601719, | |
| "step": 348 | |
| }, | |
| { | |
| "completion_length": 387.9869918823242, | |
| "epoch": 0.18666666666666668, | |
| "grad_norm": 0.06653153033495356, | |
| "kl": 0.2296142578125, | |
| "learning_rate": 6.213905926255697e-08, | |
| "loss": 0.0002, | |
| "reward": 1.3802083693444729, | |
| "reward_std": 0.2578708101063967, | |
| "rewards/equation_reward_func": 0.447916679084301, | |
| "rewards/format_reward_func": 0.9322916865348816, | |
| "step": 350 | |
| }, | |
| { | |
| "completion_length": 372.0234489440918, | |
| "epoch": 0.18773333333333334, | |
| "grad_norm": 0.05318954411938457, | |
| "kl": 0.2265625, | |
| "learning_rate": 5.978156821084987e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4192708805203438, | |
| "reward_std": 0.22500368021428585, | |
| "rewards/equation_reward_func": 0.48177084513008595, | |
| "rewards/format_reward_func": 0.9375000223517418, | |
| "step": 352 | |
| }, | |
| { | |
| "completion_length": 378.95573711395264, | |
| "epoch": 0.1888, | |
| "grad_norm": 0.09576863871090707, | |
| "kl": 0.21533203125, | |
| "learning_rate": 5.7463580319254853e-08, | |
| "loss": 0.0002, | |
| "reward": 1.3489583730697632, | |
| "reward_std": 0.28309898544102907, | |
| "rewards/equation_reward_func": 0.4296875128056854, | |
| "rewards/format_reward_func": 0.9192708507180214, | |
| "step": 354 | |
| }, | |
| { | |
| "completion_length": 432.41407585144043, | |
| "epoch": 0.18986666666666666, | |
| "grad_norm": 0.06044281622916199, | |
| "kl": 0.22998046875, | |
| "learning_rate": 5.518557697039081e-08, | |
| "loss": 0.0002, | |
| "reward": 1.2473958544433117, | |
| "reward_std": 0.25924154790118337, | |
| "rewards/equation_reward_func": 0.33072917675599456, | |
| "rewards/format_reward_func": 0.9166666902601719, | |
| "step": 356 | |
| }, | |
| { | |
| "completion_length": 374.1562614440918, | |
| "epoch": 0.19093333333333334, | |
| "grad_norm": 0.058655665655735524, | |
| "kl": 0.2113037109375, | |
| "learning_rate": 5.294803124318145e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4166666977107525, | |
| "reward_std": 0.25626226468011737, | |
| "rewards/equation_reward_func": 0.49739584885537624, | |
| "rewards/format_reward_func": 0.9192708507180214, | |
| "step": 358 | |
| }, | |
| { | |
| "completion_length": 351.77865505218506, | |
| "epoch": 0.192, | |
| "grad_norm": 0.06567119738094425, | |
| "kl": 0.23236083984375, | |
| "learning_rate": 5.07514078146106e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4088542237877846, | |
| "reward_std": 0.25361517537385225, | |
| "rewards/equation_reward_func": 0.4791666781529784, | |
| "rewards/format_reward_func": 0.9296875260770321, | |
| "step": 360 | |
| }, | |
| { | |
| "completion_length": 406.1432418823242, | |
| "epoch": 0.19306666666666666, | |
| "grad_norm": 0.07267052663236441, | |
| "kl": 0.23291015625, | |
| "learning_rate": 4.859616286322094e-08, | |
| "loss": 0.0002, | |
| "reward": 1.2812500409781933, | |
| "reward_std": 0.24471126962453127, | |
| "rewards/equation_reward_func": 0.36458334093913436, | |
| "rewards/format_reward_func": 0.9166666865348816, | |
| "step": 362 | |
| }, | |
| { | |
| "completion_length": 337.8255281448364, | |
| "epoch": 0.19413333333333332, | |
| "grad_norm": 0.06059074727025728, | |
| "kl": 0.22857666015625, | |
| "learning_rate": 4.648274397437829e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4322916977107525, | |
| "reward_std": 0.2096690242178738, | |
| "rewards/equation_reward_func": 0.4869791753590107, | |
| "rewards/format_reward_func": 0.9453125149011612, | |
| "step": 364 | |
| }, | |
| { | |
| "completion_length": 355.0833463668823, | |
| "epoch": 0.1952, | |
| "grad_norm": 0.06662905169727436, | |
| "kl": 0.2298583984375, | |
| "learning_rate": 4.4411590047320617e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4192708656191826, | |
| "reward_std": 0.22843788238242269, | |
| "rewards/equation_reward_func": 0.4739583458285779, | |
| "rewards/format_reward_func": 0.9453125149011612, | |
| "step": 366 | |
| }, | |
| { | |
| "completion_length": 328.96875858306885, | |
| "epoch": 0.19626666666666667, | |
| "grad_norm": 0.06368685219881605, | |
| "kl": 0.23321533203125, | |
| "learning_rate": 4.2383131204010494e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4817708730697632, | |
| "reward_std": 0.21312271943315864, | |
| "rewards/equation_reward_func": 0.5260416753590107, | |
| "rewards/format_reward_func": 0.9557291828095913, | |
| "step": 368 | |
| }, | |
| { | |
| "completion_length": 379.4323081970215, | |
| "epoch": 0.19733333333333333, | |
| "grad_norm": 0.07724661574230342, | |
| "kl": 0.25531005859375, | |
| "learning_rate": 4.039778869981064e-08, | |
| "loss": 0.0003, | |
| "reward": 1.398437537252903, | |
| "reward_std": 0.2854688395746052, | |
| "rewards/equation_reward_func": 0.471354179084301, | |
| "rewards/format_reward_func": 0.9270833544433117, | |
| "step": 370 | |
| }, | |
| { | |
| "completion_length": 353.55469703674316, | |
| "epoch": 0.1984, | |
| "grad_norm": 0.07167353344709883, | |
| "kl": 0.2392578125, | |
| "learning_rate": 3.845597483600049e-08, | |
| "loss": 0.0002, | |
| "reward": 1.442708358168602, | |
| "reward_std": 0.2597113102674484, | |
| "rewards/equation_reward_func": 0.49218751303851604, | |
| "rewards/format_reward_func": 0.9505208469927311, | |
| "step": 372 | |
| }, | |
| { | |
| "completion_length": 353.1119899749756, | |
| "epoch": 0.19946666666666665, | |
| "grad_norm": 0.0708211119209574, | |
| "kl": 0.22540283203125, | |
| "learning_rate": 3.655809287415284e-08, | |
| "loss": 0.0002, | |
| "reward": 1.390625037252903, | |
| "reward_std": 0.21895761415362358, | |
| "rewards/equation_reward_func": 0.4609375149011612, | |
| "rewards/format_reward_func": 0.9296875186264515, | |
| "step": 374 | |
| }, | |
| { | |
| "completion_length": 344.9635534286499, | |
| "epoch": 0.20053333333333334, | |
| "grad_norm": 0.07127844870717463, | |
| "kl": 0.21990966796875, | |
| "learning_rate": 3.4704536952387285e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4479166939854622, | |
| "reward_std": 0.22092114947736263, | |
| "rewards/equation_reward_func": 0.5000000081490725, | |
| "rewards/format_reward_func": 0.9479166828095913, | |
| "step": 376 | |
| }, | |
| { | |
| "completion_length": 321.70574045181274, | |
| "epoch": 0.2016, | |
| "grad_norm": 0.06011291534205562, | |
| "kl": 0.2210693359375, | |
| "learning_rate": 3.2895692003518575e-08, | |
| "loss": 0.0002, | |
| "reward": 1.531250037252903, | |
| "reward_std": 0.22229426866397262, | |
| "rewards/equation_reward_func": 0.5781250149011612, | |
| "rewards/format_reward_func": 0.9531250111758709, | |
| "step": 378 | |
| }, | |
| { | |
| "completion_length": 370.15625858306885, | |
| "epoch": 0.20266666666666666, | |
| "grad_norm": 0.04272961400623626, | |
| "kl": 0.2320556640625, | |
| "learning_rate": 3.113193367511635e-08, | |
| "loss": 0.0002, | |
| "reward": 1.416666716337204, | |
| "reward_std": 0.22048755269497633, | |
| "rewards/equation_reward_func": 0.47135417722165585, | |
| "rewards/format_reward_func": 0.9453125186264515, | |
| "step": 380 | |
| }, | |
| { | |
| "completion_length": 351.1224060058594, | |
| "epoch": 0.20373333333333332, | |
| "grad_norm": 0.08078010536227849, | |
| "kl": 0.23046875, | |
| "learning_rate": 2.9413628251493934e-08, | |
| "loss": 0.0002, | |
| "reward": 1.3645833693444729, | |
| "reward_std": 0.25873514311388135, | |
| "rewards/equation_reward_func": 0.4296875118743628, | |
| "rewards/format_reward_func": 0.9348958544433117, | |
| "step": 382 | |
| }, | |
| { | |
| "completion_length": 359.92188453674316, | |
| "epoch": 0.2048, | |
| "grad_norm": 0.07094226775506968, | |
| "kl": 0.2210693359375, | |
| "learning_rate": 2.774113257764066e-08, | |
| "loss": 0.0002, | |
| "reward": 1.3671875335276127, | |
| "reward_std": 0.185112988576293, | |
| "rewards/equation_reward_func": 0.42447918257676065, | |
| "rewards/format_reward_func": 0.9427083544433117, | |
| "step": 384 | |
| }, | |
| { | |
| "completion_length": 419.4531354904175, | |
| "epoch": 0.20586666666666667, | |
| "grad_norm": 0.072830678860199, | |
| "kl": 0.21502685546875, | |
| "learning_rate": 2.611479398511518e-08, | |
| "loss": 0.0002, | |
| "reward": 1.200520884245634, | |
| "reward_std": 0.23240978084504604, | |
| "rewards/equation_reward_func": 0.2838541732635349, | |
| "rewards/format_reward_func": 0.9166666865348816, | |
| "step": 386 | |
| }, | |
| { | |
| "completion_length": 366.7500114440918, | |
| "epoch": 0.20693333333333333, | |
| "grad_norm": 0.050662145843340674, | |
| "kl": 0.216552734375, | |
| "learning_rate": 2.4534950219914057e-08, | |
| "loss": 0.0002, | |
| "reward": 1.408854205161333, | |
| "reward_std": 0.23854162776842713, | |
| "rewards/equation_reward_func": 0.4687500139698386, | |
| "rewards/format_reward_func": 0.9401041865348816, | |
| "step": 388 | |
| }, | |
| { | |
| "completion_length": 380.35157203674316, | |
| "epoch": 0.208, | |
| "grad_norm": 0.051064268897514344, | |
| "kl": 0.2705078125, | |
| "learning_rate": 2.300192937233128e-08, | |
| "loss": 0.0003, | |
| "reward": 1.359375037252903, | |
| "reward_std": 0.2725842115469277, | |
| "rewards/equation_reward_func": 0.4244791741948575, | |
| "rewards/format_reward_func": 0.9348958507180214, | |
| "step": 390 | |
| }, | |
| { | |
| "completion_length": 352.7786560058594, | |
| "epoch": 0.20906666666666668, | |
| "grad_norm": 0.051418291777279995, | |
| "kl": 0.22625732421875, | |
| "learning_rate": 2.1516049808822935e-08, | |
| "loss": 0.0002, | |
| "reward": 1.419270858168602, | |
| "reward_std": 0.22951431525871158, | |
| "rewards/equation_reward_func": 0.4817708432674408, | |
| "rewards/format_reward_func": 0.9375000149011612, | |
| "step": 392 | |
| }, | |
| { | |
| "completion_length": 303.62240409851074, | |
| "epoch": 0.21013333333333334, | |
| "grad_norm": 0.06926249514713737, | |
| "kl": 0.23248291015625, | |
| "learning_rate": 2.007762010589098e-08, | |
| "loss": 0.0002, | |
| "reward": 1.5781250409781933, | |
| "reward_std": 0.2412888752296567, | |
| "rewards/equation_reward_func": 0.6093750093132257, | |
| "rewards/format_reward_func": 0.9687500186264515, | |
| "step": 394 | |
| }, | |
| { | |
| "completion_length": 307.80470085144043, | |
| "epoch": 0.2112, | |
| "grad_norm": 0.05290677595416592, | |
| "kl": 0.23150634765625, | |
| "learning_rate": 1.8686938986000627e-08, | |
| "loss": 0.0002, | |
| "reward": 1.5520833730697632, | |
| "reward_std": 0.15929541597142816, | |
| "rewards/equation_reward_func": 0.6015625111758709, | |
| "rewards/format_reward_func": 0.9505208469927311, | |
| "step": 396 | |
| }, | |
| { | |
| "completion_length": 414.739595413208, | |
| "epoch": 0.21226666666666666, | |
| "grad_norm": 0.06786874014897348, | |
| "kl": 0.218505859375, | |
| "learning_rate": 1.734429525554365e-08, | |
| "loss": 0.0002, | |
| "reward": 1.234375037252903, | |
| "reward_std": 0.2634340273216367, | |
| "rewards/equation_reward_func": 0.3098958390764892, | |
| "rewards/format_reward_func": 0.9244791828095913, | |
| "step": 398 | |
| }, | |
| { | |
| "completion_length": 344.98438453674316, | |
| "epoch": 0.21333333333333335, | |
| "grad_norm": 0.0780606363217758, | |
| "kl": 0.22955322265625, | |
| "learning_rate": 1.604996774486145e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4088541939854622, | |
| "reward_std": 0.20924207009375095, | |
| "rewards/equation_reward_func": 0.4817708465270698, | |
| "rewards/format_reward_func": 0.9270833507180214, | |
| "step": 400 | |
| }, | |
| { | |
| "completion_length": 303.3515691757202, | |
| "epoch": 0.2144, | |
| "grad_norm": 0.05979902811993369, | |
| "kl": 0.23193359375, | |
| "learning_rate": 1.4804225250339281e-08, | |
| "loss": 0.0002, | |
| "reward": 1.554687537252903, | |
| "reward_std": 0.17954271519556642, | |
| "rewards/equation_reward_func": 0.5963541828095913, | |
| "rewards/format_reward_func": 0.9583333432674408, | |
| "step": 402 | |
| }, | |
| { | |
| "completion_length": 319.5260486602783, | |
| "epoch": 0.21546666666666667, | |
| "grad_norm": 0.05068943319691407, | |
| "kl": 0.22406005859375, | |
| "learning_rate": 1.360732647858498e-08, | |
| "loss": 0.0002, | |
| "reward": 1.468750037252903, | |
| "reward_std": 0.20022964850068092, | |
| "rewards/equation_reward_func": 0.5182291793171316, | |
| "rewards/format_reward_func": 0.9505208469927311, | |
| "step": 404 | |
| }, | |
| { | |
| "completion_length": 324.12761306762695, | |
| "epoch": 0.21653333333333333, | |
| "grad_norm": 0.06110821283391216, | |
| "kl": 0.2471923828125, | |
| "learning_rate": 1.2459519992702311e-08, | |
| "loss": 0.0002, | |
| "reward": 1.4270833730697632, | |
| "reward_std": 0.2135390224866569, | |
| "rewards/equation_reward_func": 0.4869791828095913, | |
| "rewards/format_reward_func": 0.9401041828095913, | |
| "step": 406 | |
| }, | |
| { | |
| "completion_length": 380.9479274749756, | |
| "epoch": 0.2176, | |
| "grad_norm": 0.0732895161490559, | |
| "kl": 0.22747802734375, | |
| "learning_rate": 1.1361044160671629e-08, | |
| "loss": 0.0002, | |
| "reward": 1.3098958656191826, | |
| "reward_std": 0.28079350711777806, | |
| "rewards/equation_reward_func": 0.3880208439659327, | |
| "rewards/format_reward_func": 0.9218750149011612, | |
| "step": 408 | |
| }, | |
| { | |
| "completion_length": 353.096360206604, | |
| "epoch": 0.21866666666666668, | |
| "grad_norm": 0.06178117229748576, | |
| "kl": 0.2178955078125, | |
| "learning_rate": 1.0312127105846947e-08, | |
| "loss": 0.0002, | |
| "reward": 1.429687526077032, | |
| "reward_std": 0.21489776158705354, | |
| "rewards/equation_reward_func": 0.4947916818782687, | |
| "rewards/format_reward_func": 0.9348958544433117, | |
| "step": 410 | |
| }, | |
| { | |
| "completion_length": 342.0286531448364, | |
| "epoch": 0.21973333333333334, | |
| "grad_norm": 0.05874656247626641, | |
| "kl": 0.22021484375, | |
| "learning_rate": 9.312986659581301e-09, | |
| "loss": 0.0002, | |
| "reward": 1.4609375335276127, | |
| "reward_std": 0.21511370548978448, | |
| "rewards/equation_reward_func": 0.5078125125728548, | |
| "rewards/format_reward_func": 0.9531250111758709, | |
| "step": 412 | |
| }, | |
| { | |
| "completion_length": 369.84375953674316, | |
| "epoch": 0.2208, | |
| "grad_norm": 0.058447189530812425, | |
| "kl": 0.22003173828125, | |
| "learning_rate": 8.363830315988945e-09, | |
| "loss": 0.0002, | |
| "reward": 1.3776042126119137, | |
| "reward_std": 0.26471959287300706, | |
| "rewards/equation_reward_func": 0.44531251420266926, | |
| "rewards/format_reward_func": 0.9322916865348816, | |
| "step": 414 | |
| }, | |
| { | |
| "completion_length": 350.1510524749756, | |
| "epoch": 0.22186666666666666, | |
| "grad_norm": 0.048427392921259256, | |
| "kl": 0.2200927734375, | |
| "learning_rate": 7.46485518885462e-09, | |
| "loss": 0.0002, | |
| "reward": 1.4166667014360428, | |
| "reward_std": 0.17606425657868385, | |
| "rewards/equation_reward_func": 0.46614584792405367, | |
| "rewards/format_reward_func": 0.9505208544433117, | |
| "step": 416 | |
| }, | |
| { | |
| "completion_length": 336.11459159851074, | |
| "epoch": 0.22293333333333334, | |
| "grad_norm": 0.12429659093217488, | |
| "kl": 0.45501708984375, | |
| "learning_rate": 6.616247970698319e-09, | |
| "loss": 0.0005, | |
| "reward": 1.4557292014360428, | |
| "reward_std": 0.20911913039162755, | |
| "rewards/equation_reward_func": 0.49218751094304025, | |
| "rewards/format_reward_func": 0.9635416828095913, | |
| "step": 418 | |
| }, | |
| { | |
| "completion_length": 324.43229961395264, | |
| "epoch": 0.224, | |
| "grad_norm": 0.047334286459684245, | |
| "kl": 0.2275390625, | |
| "learning_rate": 5.8181848940044855e-09, | |
| "loss": 0.0002, | |
| "reward": 1.489583384245634, | |
| "reward_std": 0.21246258355677128, | |
| "rewards/equation_reward_func": 0.5494791835080832, | |
| "rewards/format_reward_func": 0.940104179084301, | |
| "step": 420 | |
| }, | |
| { | |
| "completion_length": 310.03386402130127, | |
| "epoch": 0.22506666666666666, | |
| "grad_norm": 0.04303592962371828, | |
| "kl": 0.26214599609375, | |
| "learning_rate": 5.070831694623135e-09, | |
| "loss": 0.0003, | |
| "reward": 1.492187537252903, | |
| "reward_std": 0.15913208527490497, | |
| "rewards/equation_reward_func": 0.5416666828095913, | |
| "rewards/format_reward_func": 0.9505208544433117, | |
| "step": 422 | |
| }, | |
| { | |
| "completion_length": 337.96875858306885, | |
| "epoch": 0.22613333333333333, | |
| "grad_norm": 0.049308406346144966, | |
| "kl": 0.25189208984375, | |
| "learning_rate": 4.374343577351336e-09, | |
| "loss": 0.0003, | |
| "reward": 1.4817708618938923, | |
| "reward_std": 0.1850528595969081, | |
| "rewards/equation_reward_func": 0.5286458465270698, | |
| "rewards/format_reward_func": 0.9531250186264515, | |
| "step": 424 | |
| }, | |
| { | |
| "completion_length": 323.87240409851074, | |
| "epoch": 0.2272, | |
| "grad_norm": 0.06547936368098743, | |
| "kl": 0.22930908203125, | |
| "learning_rate": 3.7288651837012745e-09, | |
| "loss": 0.0002, | |
| "reward": 1.4895833805203438, | |
| "reward_std": 0.1930558760650456, | |
| "rewards/equation_reward_func": 0.5286458432674408, | |
| "rewards/format_reward_func": 0.9609375223517418, | |
| "step": 426 | |
| }, | |
| { | |
| "completion_length": 330.25000953674316, | |
| "epoch": 0.22826666666666667, | |
| "grad_norm": 0.05744752296114911, | |
| "kl": 0.2486572265625, | |
| "learning_rate": 3.134530561862081e-09, | |
| "loss": 0.0002, | |
| "reward": 1.447916705161333, | |
| "reward_std": 0.20390508184209466, | |
| "rewards/equation_reward_func": 0.5078125167638063, | |
| "rewards/format_reward_func": 0.9401041828095913, | |
| "step": 428 | |
| }, | |
| { | |
| "completion_length": 362.11198902130127, | |
| "epoch": 0.22933333333333333, | |
| "grad_norm": 0.6353248857912718, | |
| "kl": 0.274658203125, | |
| "learning_rate": 2.5914631388619103e-09, | |
| "loss": 0.0003, | |
| "reward": 1.3984375484287739, | |
| "reward_std": 0.23770342068746686, | |
| "rewards/equation_reward_func": 0.46354167629033327, | |
| "rewards/format_reward_func": 0.9348958544433117, | |
| "step": 430 | |
| }, | |
| { | |
| "completion_length": 334.4114694595337, | |
| "epoch": 0.2304, | |
| "grad_norm": 0.03552529000856894, | |
| "kl": 0.23541259765625, | |
| "learning_rate": 2.0997756949353297e-09, | |
| "loss": 0.0002, | |
| "reward": 1.463541716337204, | |
| "reward_std": 0.12721211137250066, | |
| "rewards/equation_reward_func": 0.5078125139698386, | |
| "rewards/format_reward_func": 0.9557291753590107, | |
| "step": 432 | |
| }, | |
| { | |
| "completion_length": 322.2421941757202, | |
| "epoch": 0.23146666666666665, | |
| "grad_norm": 0.1184771152824792, | |
| "kl": 0.2554931640625, | |
| "learning_rate": 1.6595703401020844e-09, | |
| "loss": 0.0003, | |
| "reward": 1.5520833805203438, | |
| "reward_std": 0.19193315412849188, | |
| "rewards/equation_reward_func": 0.5781250193249434, | |
| "rewards/format_reward_func": 0.9739583432674408, | |
| "step": 434 | |
| }, | |
| { | |
| "completion_length": 378.0963668823242, | |
| "epoch": 0.23253333333333334, | |
| "grad_norm": 0.09243641947208214, | |
| "kl": 0.2398681640625, | |
| "learning_rate": 1.2709384929615596e-09, | |
| "loss": 0.0002, | |
| "reward": 1.3046875447034836, | |
| "reward_std": 0.3152546938508749, | |
| "rewards/equation_reward_func": 0.39322917722165585, | |
| "rewards/format_reward_func": 0.9114583469927311, | |
| "step": 436 | |
| }, | |
| { | |
| "completion_length": 309.6953182220459, | |
| "epoch": 0.2336, | |
| "grad_norm": 0.07598094346684453, | |
| "kl": 0.2366943359375, | |
| "learning_rate": 9.339608617077165e-10, | |
| "loss": 0.0002, | |
| "reward": 1.552083358168602, | |
| "reward_std": 0.17714412650093436, | |
| "rewards/equation_reward_func": 0.5859375167638063, | |
| "rewards/format_reward_func": 0.966145858168602, | |
| "step": 438 | |
| }, | |
| { | |
| "completion_length": 368.661470413208, | |
| "epoch": 0.23466666666666666, | |
| "grad_norm": 0.05002390761588348, | |
| "kl": 0.22613525390625, | |
| "learning_rate": 6.487074273681114e-10, | |
| "loss": 0.0002, | |
| "reward": 1.3645833656191826, | |
| "reward_std": 0.2412293879315257, | |
| "rewards/equation_reward_func": 0.4322916818782687, | |
| "rewards/format_reward_func": 0.9322916902601719, | |
| "step": 440 | |
| }, | |
| { | |
| "completion_length": 360.5390682220459, | |
| "epoch": 0.23573333333333332, | |
| "grad_norm": 0.051149863218837494, | |
| "kl": 0.21319580078125, | |
| "learning_rate": 4.152374292708538e-10, | |
| "loss": 0.0002, | |
| "reward": 1.3932292088866234, | |
| "reward_std": 0.20354228280484676, | |
| "rewards/equation_reward_func": 0.4505208507180214, | |
| "rewards/format_reward_func": 0.9427083544433117, | |
| "step": 442 | |
| }, | |
| { | |
| "completion_length": 341.92188358306885, | |
| "epoch": 0.2368, | |
| "grad_norm": 0.09715828568262551, | |
| "kl": 0.22161865234375, | |
| "learning_rate": 2.3359935274214204e-10, | |
| "loss": 0.0002, | |
| "reward": 1.4427083879709244, | |
| "reward_std": 0.21396331349387765, | |
| "rewards/equation_reward_func": 0.5026041809469461, | |
| "rewards/format_reward_func": 0.9401041865348816, | |
| "step": 444 | |
| }, | |
| { | |
| "completion_length": 347.1328191757202, | |
| "epoch": 0.23786666666666667, | |
| "grad_norm": 0.07462412532953941, | |
| "kl": 0.224853515625, | |
| "learning_rate": 1.0383091903720665e-10, | |
| "loss": 0.0002, | |
| "reward": 1.4479167088866234, | |
| "reward_std": 0.235118237324059, | |
| "rewards/equation_reward_func": 0.5104166835080832, | |
| "rewards/format_reward_func": 0.9375000186264515, | |
| "step": 446 | |
| }, | |
| { | |
| "completion_length": 384.0625066757202, | |
| "epoch": 0.23893333333333333, | |
| "grad_norm": 0.071734711772439, | |
| "kl": 0.25457763671875, | |
| "learning_rate": 2.595907750671533e-11, | |
| "loss": 0.0003, | |
| "reward": 1.2812500298023224, | |
| "reward_std": 0.2581098172813654, | |
| "rewards/equation_reward_func": 0.36979167303070426, | |
| "rewards/format_reward_func": 0.911458358168602, | |
| "step": 448 | |
| }, | |
| { | |
| "completion_length": 322.7994909286499, | |
| "epoch": 0.24, | |
| "grad_norm": 0.07384458300972663, | |
| "kl": 0.23516845703125, | |
| "learning_rate": 0.0, | |
| "loss": 0.0002, | |
| "reward": 1.4713541977107525, | |
| "reward_std": 0.23386369226500392, | |
| "rewards/equation_reward_func": 0.5156250118743628, | |
| "rewards/format_reward_func": 0.955729179084301, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.24053333333333332, | |
| "step": 451, | |
| "total_flos": 0.0, | |
| "train_loss": 0.0, | |
| "train_runtime": 1.3116, | |
| "train_samples_per_second": 8234.047, | |
| "train_steps_per_second": 343.085 | |
| } | |
| ], | |
| "logging_steps": 2, | |
| "max_steps": 450, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 450, | |
| "save_steps": 25, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |