{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.04094362255097961, "eval_steps": 64, "global_step": 256, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "completion_length": 52.6875, "epoch": 0.0001599360255897641, "grad_norm": 6.5625, "kl": 0.0, "learning_rate": 1.1538461538461539e-07, "loss": -0.0, "reward": 0.359375, "reward_std": 0.3756504710763693, "rewards/accuracy_reward": 0.171875, "rewards/format_reward": 0.1875, "step": 1 }, { "completion_length": 49.2109375, "epoch": 0.0007996801279488205, "grad_norm": 5.3125, "kl": 0.0012457009133868269, "learning_rate": 5.76923076923077e-07, "loss": 0.0, "reward": 0.48828125, "reward_std": 0.42536891577765346, "rewards/accuracy_reward": 0.28515625, "rewards/format_reward": 0.203125, "step": 5 }, { "completion_length": 53.75625, "epoch": 0.001599360255897641, "grad_norm": 7.15625, "kl": 0.0015386140597911436, "learning_rate": 1.153846153846154e-06, "loss": 0.0001, "reward": 0.46875, "reward_std": 0.4507805652916431, "rewards/accuracy_reward": 0.228125, "rewards/format_reward": 0.240625, "step": 10 }, { "completion_length": 56.421875, "epoch": 0.0023990403838464614, "grad_norm": 6.90625, "kl": 0.0025526953544613208, "learning_rate": 1.7307692307692306e-06, "loss": 0.0001, "reward": 0.51875, "reward_std": 0.47729706913232806, "rewards/accuracy_reward": 0.303125, "rewards/format_reward": 0.215625, "step": 15 }, { "completion_length": 50.725, "epoch": 0.003198720511795282, "grad_norm": 5.4375, "kl": 0.007305113899565186, "learning_rate": 2.307692307692308e-06, "loss": 0.0003, "reward": 0.584375, "reward_std": 0.4640388172119856, "rewards/accuracy_reward": 0.3, "rewards/format_reward": 0.284375, "step": 20 }, { "completion_length": 48.2625, "epoch": 0.003998400639744102, "grad_norm": 7.59375, "kl": 0.021931850849068724, "learning_rate": 2.884615384615385e-06, "loss": 0.0009, "reward": 0.709375, "reward_std": 0.49055532105267047, "rewards/accuracy_reward": 0.35, "rewards/format_reward": 0.359375, "step": 25 }, { "completion_length": 46.578125, "epoch": 0.004798080767692923, "grad_norm": 5.375, "kl": 0.08235460844589397, "learning_rate": 2.997761705224242e-06, "loss": 0.0033, "reward": 0.840625, "reward_std": 0.5524271633476019, "rewards/accuracy_reward": 0.384375, "rewards/format_reward": 0.45625, "step": 30 }, { "completion_length": 50.603125, "epoch": 0.005597760895641743, "grad_norm": 5.625, "kl": 0.12322683255188167, "learning_rate": 2.9886800800368023e-06, "loss": 0.0049, "reward": 1.01875, "reward_std": 0.47729706913232806, "rewards/accuracy_reward": 0.39375, "rewards/format_reward": 0.625, "step": 35 }, { "completion_length": 51.74375, "epoch": 0.006397441023590564, "grad_norm": 5.25, "kl": 0.12255409228382633, "learning_rate": 2.972657541113338e-06, "loss": 0.0049, "reward": 1.05, "reward_std": 0.4596193999052048, "rewards/accuracy_reward": 0.403125, "rewards/format_reward": 0.646875, "step": 40 }, { "completion_length": 51.15, "epoch": 0.007197121151539384, "grad_norm": 4.34375, "kl": 0.11809529251186177, "learning_rate": 2.949768792926617e-06, "loss": 0.0047, "reward": 1.1625, "reward_std": 0.5303300768136978, "rewards/accuracy_reward": 0.45, "rewards/format_reward": 0.7125, "step": 45 }, { "completion_length": 52.084375, "epoch": 0.007996801279488205, "grad_norm": 5.09375, "kl": 0.14425031405407934, "learning_rate": 2.9201205533865656e-06, "loss": 0.0058, "reward": 1.096875, "reward_std": 0.5170718248933553, "rewards/accuracy_reward": 0.41875, "rewards/format_reward": 0.678125, "step": 50 }, { "completion_length": 49.265625, "epoch": 0.008796481407437025, "grad_norm": 5.25, "kl": 0.12807998866774142, "learning_rate": 2.8838510562721076e-06, "loss": 0.0051, "reward": 1.08125, "reward_std": 0.5038135729730129, "rewards/accuracy_reward": 0.396875, "rewards/format_reward": 0.684375, "step": 55 }, { "completion_length": 53.115625, "epoch": 0.009596161535385846, "grad_norm": 5.09375, "kl": 0.12884393450804055, "learning_rate": 2.8411294067214768e-06, "loss": 0.0052, "reward": 1.125, "reward_std": 0.5391689114272594, "rewards/accuracy_reward": 0.45, "rewards/format_reward": 0.675, "step": 60 }, { "epoch": 0.010235905637744902, "eval_completion_length": 51.59410112359551, "eval_kl": 0.12826356122249297, "eval_loss": 0.004409829154610634, "eval_reward": 1.0603932584269662, "eval_reward_std": 0.5144400355521213, "eval_rewards/accuracy_reward": 0.43820224719101125, "eval_rewards/format_reward": 0.6221910112359551, "eval_runtime": 19.3936, "eval_samples_per_second": 5.156, "eval_steps_per_second": 1.289, "step": 64 }, { "completion_length": 48.90625, "epoch": 0.010395841663334666, "grad_norm": 4.09375, "kl": 0.1414291060063988, "learning_rate": 2.792154792786e-06, "loss": 0.0055, "reward": 1.03125, "reward_std": 0.4419417306780815, "rewards/accuracy_reward": 0.40625, "rewards/format_reward": 0.625, "step": 65 }, { "completion_length": 49.440625, "epoch": 0.011195521791283487, "grad_norm": 5.25, "kl": 0.11723160503897816, "learning_rate": 2.737155556723453e-06, "loss": 0.0047, "reward": 1.053125, "reward_std": 0.49055532105267047, "rewards/accuracy_reward": 0.38125, "rewards/format_reward": 0.671875, "step": 70 }, { "completion_length": 51.290625, "epoch": 0.011995201919232307, "grad_norm": 4.53125, "kl": 0.12841552164172754, "learning_rate": 2.676388130361047e-06, "loss": 0.0051, "reward": 1.084375, "reward_std": 0.5170718248933553, "rewards/accuracy_reward": 0.446875, "rewards/format_reward": 0.6375, "step": 75 }, { "completion_length": 51.165625, "epoch": 0.012794882047181128, "grad_norm": 4.4375, "kl": 0.12180936294025742, "learning_rate": 2.610135839491878e-06, "loss": 0.0049, "reward": 1.053125, "reward_std": 0.5347494941204787, "rewards/accuracy_reward": 0.4375, "rewards/format_reward": 0.615625, "step": 80 }, { "completion_length": 47.96875, "epoch": 0.013594562175129948, "grad_norm": 5.28125, "kl": 0.14888262754539028, "learning_rate": 2.538707582879289e-06, "loss": 0.006, "reward": 1.134375, "reward_std": 0.5877825018018484, "rewards/accuracy_reward": 0.4625, "rewards/format_reward": 0.671875, "step": 85 }, { "completion_length": 47.74375, "epoch": 0.014394242303078768, "grad_norm": 4.84375, "kl": 0.12864484217716382, "learning_rate": 2.4624363920282416e-06, "loss": 0.0051, "reward": 1.0875, "reward_std": 0.6187184229493141, "rewards/accuracy_reward": 0.440625, "rewards/format_reward": 0.646875, "step": 90 }, { "completion_length": 46.46875, "epoch": 0.015193922431027589, "grad_norm": 4.1875, "kl": 0.12670737880980595, "learning_rate": 2.3816778784387097e-06, "loss": 0.0051, "reward": 1.11875, "reward_std": 0.5038135729730129, "rewards/accuracy_reward": 0.434375, "rewards/format_reward": 0.684375, "step": 95 }, { "completion_length": 48.203125, "epoch": 0.01599360255897641, "grad_norm": 4.84375, "kl": 0.13195679314667358, "learning_rate": 2.296808575580705e-06, "loss": 0.0053, "reward": 1.121875, "reward_std": 0.5347494941204787, "rewards/accuracy_reward": 0.434375, "rewards/format_reward": 0.6875, "step": 100 }, { "completion_length": 57.7125, "epoch": 0.01679328268692523, "grad_norm": 4.375, "kl": 0.13236376940039918, "learning_rate": 2.208224183321428e-06, "loss": 0.0053, "reward": 1.1875, "reward_std": 0.49497473835945127, "rewards/accuracy_reward": 0.49375, "rewards/format_reward": 0.69375, "step": 105 }, { "completion_length": 47.271875, "epoch": 0.01759296281487405, "grad_norm": 5.1875, "kl": 0.1403195610502735, "learning_rate": 2.1163377229898226e-06, "loss": 0.0056, "reward": 1.190625, "reward_std": 0.49939415566623213, "rewards/accuracy_reward": 0.50625, "rewards/format_reward": 0.684375, "step": 110 }, { "completion_length": 50.625, "epoch": 0.018392642942822873, "grad_norm": 6.34375, "kl": 0.12818894329247996, "learning_rate": 2.0215776116804834e-06, "loss": 0.0051, "reward": 1.175, "reward_std": 0.548007746040821, "rewards/accuracy_reward": 0.46875, "rewards/format_reward": 0.70625, "step": 115 }, { "completion_length": 49.83125, "epoch": 0.01919232307077169, "grad_norm": 5.28125, "kl": 0.1347155719064176, "learning_rate": 1.924385664775395e-06, "loss": 0.0054, "reward": 1.065625, "reward_std": 0.49055532105267047, "rewards/accuracy_reward": 0.425, "rewards/format_reward": 0.640625, "step": 120 }, { "completion_length": 51.440625, "epoch": 0.019992003198720514, "grad_norm": 5.34375, "kl": 0.14926065368345007, "learning_rate": 1.8252150359966712e-06, "loss": 0.006, "reward": 1.125, "reward_std": 0.49497473835945127, "rewards/accuracy_reward": 0.46875, "rewards/format_reward": 0.65625, "step": 125 }, { "epoch": 0.020471811275489804, "eval_completion_length": 49.375, "eval_kl": 0.1358852016114413, "eval_loss": 0.005289154592901468, "eval_reward": 1.1934931506849316, "eval_reward_std": 0.5254868787445434, "eval_rewards/accuracy_reward": 0.464041095890411, "eval_rewards/format_reward": 0.7294520547945206, "eval_runtime": 21.8894, "eval_samples_per_second": 4.568, "eval_steps_per_second": 1.142, "step": 128 }, { "completion_length": 47.703125, "epoch": 0.020791683326669332, "grad_norm": 5.03125, "kl": 0.1484105174895376, "learning_rate": 1.7245281045947165e-06, "loss": 0.0057, "reward": 1.15625, "reward_std": 0.59662133641541, "rewards/accuracy_reward": 0.515625, "rewards/format_reward": 0.640625, "step": 130 }, { "completion_length": 48.54375, "epoch": 0.021591363454618154, "grad_norm": 6.5, "kl": 0.1305572761921212, "learning_rate": 1.62279431952272e-06, "loss": 0.0052, "reward": 1.125, "reward_std": 0.5038135729730129, "rewards/accuracy_reward": 0.425, "rewards/format_reward": 0.7, "step": 135 }, { "completion_length": 50.9, "epoch": 0.022391043582566973, "grad_norm": 5.5, "kl": 0.13983009678777308, "learning_rate": 1.5204880106489266e-06, "loss": 0.0056, "reward": 1.071875, "reward_std": 0.5435883287340403, "rewards/accuracy_reward": 0.41875, "rewards/format_reward": 0.653125, "step": 140 }, { "completion_length": 48.909375, "epoch": 0.023190723710515795, "grad_norm": 5.375, "kl": 0.1370915047591552, "learning_rate": 1.418086177211835e-06, "loss": 0.0055, "reward": 1.128125, "reward_std": 0.525910659506917, "rewards/accuracy_reward": 0.459375, "rewards/format_reward": 0.66875, "step": 145 }, { "completion_length": 46.296875, "epoch": 0.023990403838464614, "grad_norm": 6.0, "kl": 0.14994264962151646, "learning_rate": 1.3160662638295526e-06, "loss": 0.006, "reward": 1.18125, "reward_std": 0.48613590374588966, "rewards/accuracy_reward": 0.49375, "rewards/format_reward": 0.6875, "step": 150 }, { "completion_length": 50.034375, "epoch": 0.024790083966413436, "grad_norm": 4.5625, "kl": 0.13909590368857608, "learning_rate": 1.2149039344325896e-06, "loss": 0.0056, "reward": 1.0875, "reward_std": 0.49497473835945127, "rewards/accuracy_reward": 0.4125, "rewards/format_reward": 0.675, "step": 155 }, { "completion_length": 52.584375, "epoch": 0.025589764094362255, "grad_norm": 4.46875, "kl": 0.1258360159699805, "learning_rate": 1.11507085449904e-06, "loss": 0.005, "reward": 1.1625, "reward_std": 0.5656854152679444, "rewards/accuracy_reward": 0.46875, "rewards/format_reward": 0.69375, "step": 160 }, { "completion_length": 50.76875, "epoch": 0.026389444222311077, "grad_norm": 5.875, "kl": 0.17037894912064075, "learning_rate": 1.0170324919323929e-06, "loss": 0.0068, "reward": 1.16875, "reward_std": 0.5038135729730129, "rewards/accuracy_reward": 0.509375, "rewards/format_reward": 0.659375, "step": 165 }, { "completion_length": 47.490625, "epoch": 0.027189124350259896, "grad_norm": 6.1875, "kl": 0.14714236083673313, "learning_rate": 9.212459468352968e-07, "loss": 0.0059, "reward": 1.103125, "reward_std": 0.5082329902797937, "rewards/accuracy_reward": 0.425, "rewards/format_reward": 0.678125, "step": 170 }, { "completion_length": 51.25625, "epoch": 0.027988804478208718, "grad_norm": 5.53125, "kl": 0.14223471023142337, "learning_rate": 8.281578202978773e-07, "loss": 0.0057, "reward": 1.175, "reward_std": 0.5126524075865746, "rewards/accuracy_reward": 0.496875, "rewards/format_reward": 0.678125, "step": 175 }, { "completion_length": 49.021875, "epoch": 0.028788484606157537, "grad_norm": 4.28125, "kl": 0.13608955062227324, "learning_rate": 7.382021321372909e-07, "loss": 0.0054, "reward": 1.215625, "reward_std": 0.4640388172119856, "rewards/accuracy_reward": 0.51875, "rewards/format_reward": 0.696875, "step": 180 }, { "completion_length": 46.91875, "epoch": 0.02958816473410636, "grad_norm": 4.8125, "kl": 0.13916572104208172, "learning_rate": 6.517982972969913e-07, "loss": 0.0056, "reward": 1.153125, "reward_std": 0.5524271633476019, "rewards/accuracy_reward": 0.4625, "rewards/format_reward": 0.690625, "step": 185 }, { "completion_length": 47.778125, "epoch": 0.030387844862055178, "grad_norm": 4.375, "kl": 0.14800774758914487, "learning_rate": 5.693491703406479e-07, "loss": 0.0059, "reward": 1.165625, "reward_std": 0.525910659506917, "rewards/accuracy_reward": 0.478125, "rewards/format_reward": 0.6875, "step": 190 }, { "epoch": 0.030707716913234706, "eval_completion_length": 54.26535087719298, "eval_kl": 0.13607860867979757, "eval_loss": 0.00565306656062603, "eval_reward": 1.1425438596491229, "eval_reward_std": 0.49931662176784714, "eval_rewards/accuracy_reward": 0.49780701754385964, "eval_rewards/format_reward": 0.6447368421052632, "eval_runtime": 21.5975, "eval_samples_per_second": 4.63, "eval_steps_per_second": 1.158, "step": 192 }, { "completion_length": 59.322916666666664, "epoch": 0.031187524990004, "grad_norm": 6.5625, "kl": 0.14393775458059585, "learning_rate": 4.912391671582093e-07, "loss": 0.0056, "reward": 1.1614583333333333, "reward_std": 0.4345760351667802, "rewards/accuracy_reward": 0.453125, "rewards/format_reward": 0.7083333333333334, "step": 195 }, { "completion_length": 48.38125, "epoch": 0.03198720511795282, "grad_norm": 5.09375, "kl": 0.15551140770548955, "learning_rate": 4.1783247264156647e-07, "loss": 0.0062, "reward": 1.11875, "reward_std": 0.4684582345187664, "rewards/accuracy_reward": 0.440625, "rewards/format_reward": 0.678125, "step": 200 }, { "completion_length": 49.403125, "epoch": 0.03278688524590164, "grad_norm": 5.09375, "kl": 0.13503661315189674, "learning_rate": 3.494713426864761e-07, "loss": 0.0054, "reward": 1.296875, "reward_std": 0.5435883287340403, "rewards/accuracy_reward": 0.546875, "rewards/format_reward": 0.75, "step": 205 }, { "completion_length": 46.8875, "epoch": 0.03358656537385046, "grad_norm": 5.28125, "kl": 0.13574067065492273, "learning_rate": 2.86474508437579e-07, "loss": 0.0054, "reward": 1.2625, "reward_std": 0.5126524075865746, "rewards/accuracy_reward": 0.54375, "rewards/format_reward": 0.71875, "step": 210 }, { "completion_length": 49.70625, "epoch": 0.03438624550179928, "grad_norm": 4.09375, "kl": 0.12194884136551991, "learning_rate": 2.2913569021667464e-07, "loss": 0.0049, "reward": 1.203125, "reward_std": 0.5435883287340403, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.703125, "step": 215 }, { "completion_length": 49.715625, "epoch": 0.0351859256297481, "grad_norm": 5.28125, "kl": 0.14014763564337046, "learning_rate": 1.7772222806299266e-07, "loss": 0.0056, "reward": 1.203125, "reward_std": 0.4640388172119856, "rewards/accuracy_reward": 0.503125, "rewards/format_reward": 0.7, "step": 220 }, { "completion_length": 50.23125, "epoch": 0.03598560575769692, "grad_norm": 5.75, "kl": 0.13671348651405424, "learning_rate": 1.3247383527051986e-07, "loss": 0.0055, "reward": 1.209375, "reward_std": 0.48171648643910886, "rewards/accuracy_reward": 0.475, "rewards/format_reward": 0.734375, "step": 225 }, { "completion_length": 49.9875, "epoch": 0.036785285885645745, "grad_norm": 4.28125, "kl": 0.1529265999328345, "learning_rate": 9.360148073396963e-08, "loss": 0.0061, "reward": 1.13125, "reward_std": 0.5568465806543828, "rewards/accuracy_reward": 0.48125, "rewards/format_reward": 0.65, "step": 230 }, { "completion_length": 49.721875, "epoch": 0.03758496601359456, "grad_norm": 3.671875, "kl": 0.15454837685683742, "learning_rate": 6.128640531440515e-08, "loss": 0.0062, "reward": 1.109375, "reward_std": 0.49939415566623213, "rewards/accuracy_reward": 0.434375, "rewards/format_reward": 0.675, "step": 235 }, { "completion_length": 49.690625, "epoch": 0.03838464614154338, "grad_norm": 4.65625, "kl": 0.11877592669334262, "learning_rate": 3.567927681068317e-08, "loss": 0.0048, "reward": 1.2375, "reward_std": 0.5038135729730129, "rewards/accuracy_reward": 0.453125, "rewards/format_reward": 0.784375, "step": 240 }, { "completion_length": 47.890625, "epoch": 0.039184326269492205, "grad_norm": 5.53125, "kl": 0.17146191628417, "learning_rate": 1.68994874766229e-08, "loss": 0.0069, "reward": 1.15, "reward_std": 0.48613590374588966, "rewards/accuracy_reward": 0.48125, "rewards/format_reward": 0.66875, "step": 245 }, { "completion_length": 48.56875, "epoch": 0.03998400639744103, "grad_norm": 4.71875, "kl": 0.14072445079218596, "learning_rate": 5.034597359205639e-09, "loss": 0.0056, "reward": 1.2, "reward_std": 0.5391689114272594, "rewards/accuracy_reward": 0.4625, "rewards/format_reward": 0.7375, "step": 250 }, { "completion_length": 51.275, "epoch": 0.04078368652538984, "grad_norm": 4.84375, "kl": 0.14359070940408855, "learning_rate": 1.3992605321688778e-10, "loss": 0.0057, "reward": 1.209375, "reward_std": 0.4640388172119856, "rewards/accuracy_reward": 0.4875, "rewards/format_reward": 0.721875, "step": 255 }, { "epoch": 0.04094362255097961, "eval_completion_length": 51.548780487804876, "eval_kl": 0.1350904534593588, "eval_loss": 0.005155267659574747, "eval_reward": 1.1676829268292683, "eval_reward_std": 0.4958370625972748, "eval_rewards/accuracy_reward": 0.47560975609756095, "eval_rewards/format_reward": 0.6920731707317073, "eval_runtime": 24.1797, "eval_samples_per_second": 4.136, "eval_steps_per_second": 1.034, "step": 256 }, { "epoch": 0.04094362255097961, "step": 256, "total_flos": 0.0, "train_loss": 0.004944845609998083, "train_runtime": 5152.2436, "train_samples_per_second": 3.18, "train_steps_per_second": 0.05 } ], "logging_steps": 5, "max_steps": 256, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }