| { | |
| "best_metric": 0.1417153775691986, | |
| "best_model_checkpoint": "limbxy_pose/checkpoint-1780", | |
| "epoch": 20.0, | |
| "eval_steps": 500, | |
| "global_step": 1780, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.2808988764044944, | |
| "grad_norm": 5.3874141476076276e-14, | |
| "learning_rate": 5e-06, | |
| "loss": 0.3282, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.5617977528089888, | |
| "grad_norm": 2.9326387238765017e-12, | |
| "learning_rate": 1e-05, | |
| "loss": 0.3304, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.8426966292134831, | |
| "grad_norm": 7.455580207024748e-13, | |
| "learning_rate": 1.5e-05, | |
| "loss": 0.3414, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "eval_loss": 0.33112141489982605, | |
| "eval_rmse": 0.5754314661026001, | |
| "eval_runtime": 9.4539, | |
| "eval_samples_per_second": 105.776, | |
| "eval_steps_per_second": 1.692, | |
| "step": 89 | |
| }, | |
| { | |
| "epoch": 1.1235955056179776, | |
| "grad_norm": 2.835559207142041e-13, | |
| "learning_rate": 2e-05, | |
| "loss": 0.3408, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 1.404494382022472, | |
| "grad_norm": 1.3095855780007049e-12, | |
| "learning_rate": 2.5e-05, | |
| "loss": 0.3358, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 1.6853932584269664, | |
| "grad_norm": 1.1560928603326914e-10, | |
| "learning_rate": 3e-05, | |
| "loss": 0.3348, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.9662921348314608, | |
| "grad_norm": 6.433731585681057e-12, | |
| "learning_rate": 3.5e-05, | |
| "loss": 0.3313, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "eval_loss": 0.33112141489982605, | |
| "eval_rmse": 0.5754314661026001, | |
| "eval_runtime": 9.3581, | |
| "eval_samples_per_second": 106.859, | |
| "eval_steps_per_second": 1.71, | |
| "step": 178 | |
| }, | |
| { | |
| "epoch": 2.247191011235955, | |
| "grad_norm": 654211.5, | |
| "learning_rate": 4e-05, | |
| "loss": 0.3285, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 2.5280898876404496, | |
| "grad_norm": 342525.875, | |
| "learning_rate": 4.5e-05, | |
| "loss": 0.2248, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 2.808988764044944, | |
| "grad_norm": 1131143.125, | |
| "learning_rate": 5e-05, | |
| "loss": 0.2277, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "eval_loss": 0.1668396145105362, | |
| "eval_rmse": 0.40846002101898193, | |
| "eval_runtime": 9.2408, | |
| "eval_samples_per_second": 108.216, | |
| "eval_steps_per_second": 1.731, | |
| "step": 267 | |
| }, | |
| { | |
| "epoch": 3.0898876404494384, | |
| "grad_norm": 3920230.0, | |
| "learning_rate": 4.918300653594771e-05, | |
| "loss": 0.2028, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 3.370786516853933, | |
| "grad_norm": 1259230.75, | |
| "learning_rate": 4.8366013071895424e-05, | |
| "loss": 0.2022, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 3.6516853932584272, | |
| "grad_norm": 380599.59375, | |
| "learning_rate": 4.7549019607843135e-05, | |
| "loss": 0.1957, | |
| "step": 325 | |
| }, | |
| { | |
| "epoch": 3.932584269662921, | |
| "grad_norm": 928917.8125, | |
| "learning_rate": 4.673202614379085e-05, | |
| "loss": 0.1756, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 4.0, | |
| "eval_loss": 0.15156960487365723, | |
| "eval_rmse": 0.38931941986083984, | |
| "eval_runtime": 9.3606, | |
| "eval_samples_per_second": 106.831, | |
| "eval_steps_per_second": 1.709, | |
| "step": 356 | |
| }, | |
| { | |
| "epoch": 4.213483146067416, | |
| "grad_norm": 4586462.0, | |
| "learning_rate": 4.5915032679738564e-05, | |
| "loss": 0.1686, | |
| "step": 375 | |
| }, | |
| { | |
| "epoch": 4.49438202247191, | |
| "grad_norm": 1776099.0, | |
| "learning_rate": 4.5098039215686275e-05, | |
| "loss": 0.1619, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 4.775280898876405, | |
| "grad_norm": 3756013.5, | |
| "learning_rate": 4.4281045751633986e-05, | |
| "loss": 0.1761, | |
| "step": 425 | |
| }, | |
| { | |
| "epoch": 5.0, | |
| "eval_loss": 0.1556388884782791, | |
| "eval_rmse": 0.394510954618454, | |
| "eval_runtime": 9.292, | |
| "eval_samples_per_second": 107.62, | |
| "eval_steps_per_second": 1.722, | |
| "step": 445 | |
| }, | |
| { | |
| "epoch": 5.056179775280899, | |
| "grad_norm": 2636789.0, | |
| "learning_rate": 4.3464052287581704e-05, | |
| "loss": 0.1791, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 5.337078651685394, | |
| "grad_norm": 2208706.25, | |
| "learning_rate": 4.2647058823529415e-05, | |
| "loss": 0.1736, | |
| "step": 475 | |
| }, | |
| { | |
| "epoch": 5.617977528089888, | |
| "grad_norm": 499775.21875, | |
| "learning_rate": 4.1830065359477126e-05, | |
| "loss": 0.1565, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 5.898876404494382, | |
| "grad_norm": 1655149.625, | |
| "learning_rate": 4.101307189542484e-05, | |
| "loss": 0.1632, | |
| "step": 525 | |
| }, | |
| { | |
| "epoch": 6.0, | |
| "eval_loss": 0.14596471190452576, | |
| "eval_rmse": 0.38205331563949585, | |
| "eval_runtime": 9.4973, | |
| "eval_samples_per_second": 105.293, | |
| "eval_steps_per_second": 1.685, | |
| "step": 534 | |
| }, | |
| { | |
| "epoch": 6.179775280898877, | |
| "grad_norm": 69511.53125, | |
| "learning_rate": 4.0196078431372555e-05, | |
| "loss": 0.1537, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 6.460674157303371, | |
| "grad_norm": 1434219.875, | |
| "learning_rate": 3.9379084967320266e-05, | |
| "loss": 0.1578, | |
| "step": 575 | |
| }, | |
| { | |
| "epoch": 6.741573033707866, | |
| "grad_norm": 959990.375, | |
| "learning_rate": 3.8562091503267977e-05, | |
| "loss": 0.1488, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 7.0, | |
| "eval_loss": 0.14534823596477509, | |
| "eval_rmse": 0.3812456429004669, | |
| "eval_runtime": 9.4779, | |
| "eval_samples_per_second": 105.508, | |
| "eval_steps_per_second": 1.688, | |
| "step": 623 | |
| }, | |
| { | |
| "epoch": 7.022471910112359, | |
| "grad_norm": 887092.875, | |
| "learning_rate": 3.774509803921569e-05, | |
| "loss": 0.1508, | |
| "step": 625 | |
| }, | |
| { | |
| "epoch": 7.303370786516854, | |
| "grad_norm": 1917969.375, | |
| "learning_rate": 3.6928104575163405e-05, | |
| "loss": 0.1532, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 7.584269662921348, | |
| "grad_norm": 142502.96875, | |
| "learning_rate": 3.611111111111111e-05, | |
| "loss": 0.1501, | |
| "step": 675 | |
| }, | |
| { | |
| "epoch": 7.865168539325842, | |
| "grad_norm": 308292.90625, | |
| "learning_rate": 3.529411764705883e-05, | |
| "loss": 0.1502, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 8.0, | |
| "eval_loss": 0.1449001282453537, | |
| "eval_rmse": 0.38065749406814575, | |
| "eval_runtime": 9.6336, | |
| "eval_samples_per_second": 103.804, | |
| "eval_steps_per_second": 1.661, | |
| "step": 712 | |
| }, | |
| { | |
| "epoch": 8.146067415730338, | |
| "grad_norm": 2082431.625, | |
| "learning_rate": 3.447712418300654e-05, | |
| "loss": 0.1591, | |
| "step": 725 | |
| }, | |
| { | |
| "epoch": 8.426966292134832, | |
| "grad_norm": 2036905.875, | |
| "learning_rate": 3.366013071895425e-05, | |
| "loss": 0.1563, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 8.707865168539326, | |
| "grad_norm": 77962.4765625, | |
| "learning_rate": 3.284313725490196e-05, | |
| "loss": 0.1551, | |
| "step": 775 | |
| }, | |
| { | |
| "epoch": 8.98876404494382, | |
| "grad_norm": 1343378.0, | |
| "learning_rate": 3.202614379084967e-05, | |
| "loss": 0.1512, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 9.0, | |
| "eval_loss": 0.15092459321022034, | |
| "eval_rmse": 0.38849014043807983, | |
| "eval_runtime": 9.6447, | |
| "eval_samples_per_second": 103.684, | |
| "eval_steps_per_second": 1.659, | |
| "step": 801 | |
| }, | |
| { | |
| "epoch": 9.269662921348315, | |
| "grad_norm": 2264512.0, | |
| "learning_rate": 3.120915032679739e-05, | |
| "loss": 0.1495, | |
| "step": 825 | |
| }, | |
| { | |
| "epoch": 9.55056179775281, | |
| "grad_norm": 1154337.125, | |
| "learning_rate": 3.0392156862745097e-05, | |
| "loss": 0.1536, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 9.831460674157304, | |
| "grad_norm": 1739664.875, | |
| "learning_rate": 2.957516339869281e-05, | |
| "loss": 0.1491, | |
| "step": 875 | |
| }, | |
| { | |
| "epoch": 10.0, | |
| "eval_loss": 0.1450948417186737, | |
| "eval_rmse": 0.3809131681919098, | |
| "eval_runtime": 9.7315, | |
| "eval_samples_per_second": 102.759, | |
| "eval_steps_per_second": 1.644, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 10.112359550561798, | |
| "grad_norm": 1887076.125, | |
| "learning_rate": 2.8758169934640522e-05, | |
| "loss": 0.1586, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 10.393258426966293, | |
| "grad_norm": 2062948.625, | |
| "learning_rate": 2.7941176470588236e-05, | |
| "loss": 0.1537, | |
| "step": 925 | |
| }, | |
| { | |
| "epoch": 10.674157303370787, | |
| "grad_norm": 622017.4375, | |
| "learning_rate": 2.7124183006535947e-05, | |
| "loss": 0.1586, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 10.955056179775282, | |
| "grad_norm": 279420.875, | |
| "learning_rate": 2.630718954248366e-05, | |
| "loss": 0.1615, | |
| "step": 975 | |
| }, | |
| { | |
| "epoch": 11.0, | |
| "eval_loss": 0.14514553546905518, | |
| "eval_rmse": 0.3809797465801239, | |
| "eval_runtime": 9.5399, | |
| "eval_samples_per_second": 104.823, | |
| "eval_steps_per_second": 1.677, | |
| "step": 979 | |
| }, | |
| { | |
| "epoch": 11.235955056179776, | |
| "grad_norm": 468609.03125, | |
| "learning_rate": 2.5490196078431373e-05, | |
| "loss": 0.1498, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 11.51685393258427, | |
| "grad_norm": 420918.21875, | |
| "learning_rate": 2.4673202614379087e-05, | |
| "loss": 0.1511, | |
| "step": 1025 | |
| }, | |
| { | |
| "epoch": 11.797752808988765, | |
| "grad_norm": 1311995.125, | |
| "learning_rate": 2.38562091503268e-05, | |
| "loss": 0.1549, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 12.0, | |
| "eval_loss": 0.14528058469295502, | |
| "eval_rmse": 0.38115692138671875, | |
| "eval_runtime": 9.3492, | |
| "eval_samples_per_second": 106.962, | |
| "eval_steps_per_second": 1.711, | |
| "step": 1068 | |
| }, | |
| { | |
| "epoch": 12.07865168539326, | |
| "grad_norm": 2639625.0, | |
| "learning_rate": 2.303921568627451e-05, | |
| "loss": 0.153, | |
| "step": 1075 | |
| }, | |
| { | |
| "epoch": 12.359550561797754, | |
| "grad_norm": 774080.25, | |
| "learning_rate": 2.2222222222222223e-05, | |
| "loss": 0.1482, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 12.640449438202246, | |
| "grad_norm": 55058.171875, | |
| "learning_rate": 2.1405228758169934e-05, | |
| "loss": 0.1468, | |
| "step": 1125 | |
| }, | |
| { | |
| "epoch": 12.921348314606742, | |
| "grad_norm": 386005.375, | |
| "learning_rate": 2.058823529411765e-05, | |
| "loss": 0.1511, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 13.0, | |
| "eval_loss": 0.1451251208782196, | |
| "eval_rmse": 0.380952924489975, | |
| "eval_runtime": 9.2084, | |
| "eval_samples_per_second": 108.596, | |
| "eval_steps_per_second": 1.738, | |
| "step": 1157 | |
| }, | |
| { | |
| "epoch": 13.202247191011235, | |
| "grad_norm": 1428037.875, | |
| "learning_rate": 1.977124183006536e-05, | |
| "loss": 0.1493, | |
| "step": 1175 | |
| }, | |
| { | |
| "epoch": 13.48314606741573, | |
| "grad_norm": 1060426.0, | |
| "learning_rate": 1.895424836601307e-05, | |
| "loss": 0.1475, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 13.764044943820224, | |
| "grad_norm": 393123.84375, | |
| "learning_rate": 1.8137254901960785e-05, | |
| "loss": 0.146, | |
| "step": 1225 | |
| }, | |
| { | |
| "epoch": 14.0, | |
| "eval_loss": 0.14483922719955444, | |
| "eval_rmse": 0.3805775046348572, | |
| "eval_runtime": 9.3462, | |
| "eval_samples_per_second": 106.995, | |
| "eval_steps_per_second": 1.712, | |
| "step": 1246 | |
| }, | |
| { | |
| "epoch": 14.044943820224718, | |
| "grad_norm": 55979.83203125, | |
| "learning_rate": 1.7320261437908496e-05, | |
| "loss": 0.1474, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 14.325842696629213, | |
| "grad_norm": 1408408.875, | |
| "learning_rate": 1.650326797385621e-05, | |
| "loss": 0.1483, | |
| "step": 1275 | |
| }, | |
| { | |
| "epoch": 14.606741573033707, | |
| "grad_norm": 1193337.625, | |
| "learning_rate": 1.568627450980392e-05, | |
| "loss": 0.1472, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 14.887640449438202, | |
| "grad_norm": 138167.046875, | |
| "learning_rate": 1.4869281045751634e-05, | |
| "loss": 0.1495, | |
| "step": 1325 | |
| }, | |
| { | |
| "epoch": 15.0, | |
| "eval_loss": 0.14416830241680145, | |
| "eval_rmse": 0.37969499826431274, | |
| "eval_runtime": 9.3515, | |
| "eval_samples_per_second": 106.935, | |
| "eval_steps_per_second": 1.711, | |
| "step": 1335 | |
| }, | |
| { | |
| "epoch": 15.168539325842696, | |
| "grad_norm": 1532522.0, | |
| "learning_rate": 1.4052287581699347e-05, | |
| "loss": 0.1461, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 15.44943820224719, | |
| "grad_norm": 562337.5, | |
| "learning_rate": 1.323529411764706e-05, | |
| "loss": 0.1492, | |
| "step": 1375 | |
| }, | |
| { | |
| "epoch": 15.730337078651685, | |
| "grad_norm": 814187.375, | |
| "learning_rate": 1.2418300653594772e-05, | |
| "loss": 0.1463, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 16.0, | |
| "eval_loss": 0.1432572454214096, | |
| "eval_rmse": 0.37849339842796326, | |
| "eval_runtime": 9.2916, | |
| "eval_samples_per_second": 107.624, | |
| "eval_steps_per_second": 1.722, | |
| "step": 1424 | |
| }, | |
| { | |
| "epoch": 16.01123595505618, | |
| "grad_norm": 670902.375, | |
| "learning_rate": 1.1601307189542485e-05, | |
| "loss": 0.148, | |
| "step": 1425 | |
| }, | |
| { | |
| "epoch": 16.292134831460675, | |
| "grad_norm": 1786915.75, | |
| "learning_rate": 1.0784313725490197e-05, | |
| "loss": 0.1464, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 16.573033707865168, | |
| "grad_norm": 204775.359375, | |
| "learning_rate": 9.96732026143791e-06, | |
| "loss": 0.147, | |
| "step": 1475 | |
| }, | |
| { | |
| "epoch": 16.853932584269664, | |
| "grad_norm": 434166.53125, | |
| "learning_rate": 9.150326797385621e-06, | |
| "loss": 0.1455, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 17.0, | |
| "eval_loss": 0.14278554916381836, | |
| "eval_rmse": 0.37786978483200073, | |
| "eval_runtime": 9.2131, | |
| "eval_samples_per_second": 108.541, | |
| "eval_steps_per_second": 1.737, | |
| "step": 1513 | |
| }, | |
| { | |
| "epoch": 17.134831460674157, | |
| "grad_norm": 427259.8125, | |
| "learning_rate": 8.333333333333334e-06, | |
| "loss": 0.1476, | |
| "step": 1525 | |
| }, | |
| { | |
| "epoch": 17.415730337078653, | |
| "grad_norm": 216315.09375, | |
| "learning_rate": 7.5163398692810456e-06, | |
| "loss": 0.1481, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 17.696629213483146, | |
| "grad_norm": 842487.625, | |
| "learning_rate": 6.699346405228758e-06, | |
| "loss": 0.1435, | |
| "step": 1575 | |
| }, | |
| { | |
| "epoch": 17.97752808988764, | |
| "grad_norm": 376340.8125, | |
| "learning_rate": 5.882352941176471e-06, | |
| "loss": 0.144, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 18.0, | |
| "eval_loss": 0.1422489583492279, | |
| "eval_rmse": 0.37715908885002136, | |
| "eval_runtime": 9.3723, | |
| "eval_samples_per_second": 106.697, | |
| "eval_steps_per_second": 1.707, | |
| "step": 1602 | |
| }, | |
| { | |
| "epoch": 18.258426966292134, | |
| "grad_norm": 287156.1875, | |
| "learning_rate": 5.065359477124184e-06, | |
| "loss": 0.1435, | |
| "step": 1625 | |
| }, | |
| { | |
| "epoch": 18.53932584269663, | |
| "grad_norm": 129252.7109375, | |
| "learning_rate": 4.2483660130718954e-06, | |
| "loss": 0.142, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 18.820224719101123, | |
| "grad_norm": 415142.15625, | |
| "learning_rate": 3.431372549019608e-06, | |
| "loss": 0.1461, | |
| "step": 1675 | |
| }, | |
| { | |
| "epoch": 19.0, | |
| "eval_loss": 0.14274005591869354, | |
| "eval_rmse": 0.3778095543384552, | |
| "eval_runtime": 9.4009, | |
| "eval_samples_per_second": 106.373, | |
| "eval_steps_per_second": 1.702, | |
| "step": 1691 | |
| }, | |
| { | |
| "epoch": 19.10112359550562, | |
| "grad_norm": 202907.5625, | |
| "learning_rate": 2.6143790849673204e-06, | |
| "loss": 0.1441, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 19.382022471910112, | |
| "grad_norm": 269955.5625, | |
| "learning_rate": 1.7973856209150326e-06, | |
| "loss": 0.1438, | |
| "step": 1725 | |
| }, | |
| { | |
| "epoch": 19.662921348314608, | |
| "grad_norm": 197176.828125, | |
| "learning_rate": 9.80392156862745e-07, | |
| "loss": 0.1436, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 19.9438202247191, | |
| "grad_norm": 579304.625, | |
| "learning_rate": 1.6339869281045752e-07, | |
| "loss": 0.1425, | |
| "step": 1775 | |
| }, | |
| { | |
| "epoch": 20.0, | |
| "eval_loss": 0.1417153775691986, | |
| "eval_rmse": 0.3764510452747345, | |
| "eval_runtime": 9.5809, | |
| "eval_samples_per_second": 104.375, | |
| "eval_steps_per_second": 1.67, | |
| "step": 1780 | |
| }, | |
| { | |
| "epoch": 20.0, | |
| "step": 1780, | |
| "total_flos": 0.0, | |
| "train_loss": 0.17676029650682815, | |
| "train_runtime": 3303.753, | |
| "train_samples_per_second": 34.276, | |
| "train_steps_per_second": 0.539 | |
| } | |
| ], | |
| "logging_steps": 25, | |
| "max_steps": 1780, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 20, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 64, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |