Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/gongc1995-city-university-of-hong-kong/huggingface/runs/papxj9zr) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/gongc1995-city-university-of-hong-kong/huggingface/runs/ywgmacxu) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,13 @@
 {
+    "eval_loss": 0.01594367064535618,
+    "eval_runtime": 111.2602,
+    "eval_samples": 99,
+    "eval_samples_per_second": 0.89,
+    "eval_steps_per_second": 0.018,
     "total_flos": 0.0,
-    "train_loss": 0.01271514138510042,
-    "train_runtime": 147461.7668,
-    "train_samples": 72441,
-    "train_samples_per_second": 0.491,
-    "train_steps_per_second": 0.004
+    "train_loss": -42.93539375066757,
+    "train_runtime": 225.8365,
+    "train_samples": 20,
+    "train_samples_per_second": 0.177,
+    "train_steps_per_second": 0.009
 }

eval_results.json

@@ -0,0 +1,7 @@
+{
+    "eval_loss": 0.01594367064535618,
+    "eval_runtime": 111.2602,
+    "eval_samples": 99,
+    "eval_samples_per_second": 0.89,
+    "eval_steps_per_second": 0.018
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1909a6771bf5f61c6d117bd479d14bc6a180db3ffe32bd2c09746e2d445972e
+oid sha256:61874210ca7c360f43f8c622cecc12441083d40190eae3b56bc9d6e1c0a30c1e
 size 4997744872

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:44c276968b28dc5a27fe503f3bc4fdfdfaedc77b7011657a0c1d212b54082e89
+oid sha256:c523a43b8a17269d5fab33395048a83633f4d1d89c1958570cea738e2bbe80c9
 size 4997235176

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:998814a0196413ffbdd822f2d4beb069f44ef1cf932a3aefd3bf488fa15c3baf
+oid sha256:97ae01e3519c52e63a018bca96ab17a89c4cd5cab1c6d742efed0fa5c0e2bb17
 size 3843741912

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.01271514138510042,
-    "train_runtime": 147461.7668,
-    "train_samples": 72441,
-    "train_samples_per_second": 0.491,
-    "train_steps_per_second": 0.004
+    "train_loss": -42.93539375066757,
+    "train_runtime": 225.8365,
+    "train_samples": 20,
+    "train_samples_per_second": 0.177,
+    "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -1,1794 +1,33 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9987438399845395,
+  "epoch": 0,
   "eval_steps": 100,
-  "global_step": 646,
+  "global_step": 0,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 457.96500358581545,
-      "epoch": 0.007730215479756498,
-      "grad_norm": 0.7608509038009253,
-      "kl": 0.0007220268249511719,
-      "learning_rate": 7.692307692307694e-07,
-      "loss": 0.0,
-      "reward": 0.45107144135981797,
-      "reward_std": 0.4365754766389728,
-      "rewards/accuracy_reward": 0.04785714372992515,
-      "rewards/format_reward": 0.4032142972573638,
-      "step": 5
-    },
-    {
-      "completion_length": 445.5060745239258,
-      "epoch": 0.015460430959512996,
-      "grad_norm": 1.5113540423097132,
-      "kl": 0.006823968887329101,
-      "learning_rate": 1.5384615384615387e-06,
-      "loss": 0.0003,
-      "reward": 0.5625000160187483,
-      "reward_std": 0.4424866631627083,
-      "rewards/accuracy_reward": 0.06107142954133451,
-      "rewards/format_reward": 0.5014285854995251,
-      "step": 10
-    },
-    {
-      "completion_length": 411.183215713501,
-      "epoch": 0.023190646439269495,
-      "grad_norm": 1.5933497538626709,
-      "kl": 0.029457855224609374,
-      "learning_rate": 2.307692307692308e-06,
-      "loss": 0.0012,
-      "reward": 0.7785714413970709,
-      "reward_std": 0.35438590599223974,
-      "rewards/accuracy_reward": 0.05464285819325596,
-      "rewards/format_reward": 0.7239285875111818,
-      "step": 15
-    },
-    {
-      "completion_length": 381.2946424484253,
-      "epoch": 0.03092086191902599,
-      "grad_norm": 0.4776993449956818,
-      "kl": 0.02753753662109375,
-      "learning_rate": 3.0769230769230774e-06,
-      "loss": 0.0011,
-      "reward": 0.8642857238650322,
-      "reward_std": 0.29294114708900454,
-      "rewards/accuracy_reward": 0.03821428623050451,
-      "rewards/format_reward": 0.8260714389383793,
-      "step": 20
-    },
-    {
-      "completion_length": 362.5117889404297,
-      "epoch": 0.03865107739878249,
-      "grad_norm": 0.635286407672086,
-      "kl": 0.03197479248046875,
-      "learning_rate": 3.846153846153847e-06,
-      "loss": 0.0013,
-      "reward": 0.9035714380443096,
-      "reward_std": 0.23572507686913013,
-      "rewards/accuracy_reward": 0.032142857718281445,
-      "rewards/format_reward": 0.8714285790920258,
-      "step": 25
-    },
-    {
-      "completion_length": 315.9125024795532,
-      "epoch": 0.04638129287853899,
-      "grad_norm": 0.3737341489393726,
-      "kl": 0.042462158203125,
-      "learning_rate": 4.615384615384616e-06,
-      "loss": 0.0017,
-      "reward": 0.9617857217788697,
-      "reward_std": 0.1909911316819489,
-      "rewards/accuracy_reward": 0.04142857207916677,
-      "rewards/format_reward": 0.920357147604227,
-      "step": 30
-    },
-    {
-      "completion_length": 303.57821617126467,
-      "epoch": 0.054111508358295486,
-      "grad_norm": 0.29770380771379046,
-      "kl": 0.05437469482421875,
-      "learning_rate": 5.384615384615385e-06,
-      "loss": 0.0022,
-      "reward": 0.975714298337698,
-      "reward_std": 0.16020548259839415,
-      "rewards/accuracy_reward": 0.03321428613271564,
-      "rewards/format_reward": 0.942500003427267,
-      "step": 35
-    },
-    {
-      "completion_length": 327.77964363098147,
-      "epoch": 0.06184172383805198,
-      "grad_norm": 0.4560285277274068,
-      "kl": 0.056903076171875,
-      "learning_rate": 6.153846153846155e-06,
-      "loss": 0.0023,
-      "reward": 1.016785728931427,
-      "reward_std": 0.1489170640707016,
-      "rewards/accuracy_reward": 0.05392857214901596,
-      "rewards/format_reward": 0.9628571450710297,
-      "step": 40
-    },
-    {
-      "completion_length": 365.1317882537842,
-      "epoch": 0.06957193931780849,
-      "grad_norm": 0.31719927124196806,
-      "kl": 0.0534454345703125,
-      "learning_rate": 6.923076923076923e-06,
-      "loss": 0.0021,
-      "reward": 1.0375000171363353,
-      "reward_std": 0.19086767928674817,
-      "rewards/accuracy_reward": 0.08285714453086257,
-      "rewards/format_reward": 0.9546428598463536,
-      "step": 45
-    },
-    {
-      "completion_length": 358.1814300537109,
-      "epoch": 0.07730215479756498,
-      "grad_norm": 0.38771993742220967,
-      "kl": 0.06278076171875,
-      "learning_rate": 7.692307692307694e-06,
-      "loss": 0.0025,
-      "reward": 1.040357157588005,
-      "reward_std": 0.21823968663811683,
-      "rewards/accuracy_reward": 0.0939285738626495,
-      "rewards/format_reward": 0.9464285746216774,
-      "step": 50
-    },
-    {
-      "completion_length": 338.32928562164307,
-      "epoch": 0.08503237027732148,
-      "grad_norm": 0.3009616852428193,
-      "kl": 0.07071533203125,
-      "learning_rate": 8.461538461538462e-06,
-      "loss": 0.0028,
-      "reward": 1.0582143023610115,
-      "reward_std": 0.1873143390752375,
-      "rewards/accuracy_reward": 0.09785714475437998,
-      "rewards/format_reward": 0.9603571452200412,
-      "step": 55
-    },
-    {
-      "completion_length": 355.8407180786133,
-      "epoch": 0.09276258575707798,
-      "grad_norm": 0.325964246595786,
-      "kl": 0.07974853515625,
-      "learning_rate": 9.230769230769232e-06,
-      "loss": 0.0032,
-      "reward": 1.0664285972714425,
-      "reward_std": 0.1719504727050662,
-      "rewards/accuracy_reward": 0.09464285941794515,
-      "rewards/format_reward": 0.9717857159674168,
-      "step": 60
-    },
-    {
-      "completion_length": 385.9732172012329,
-      "epoch": 0.10049280123683448,
-      "grad_norm": 0.29390759666087973,
-      "kl": 0.095562744140625,
-      "learning_rate": 1e-05,
-      "loss": 0.0038,
-      "reward": 1.1042857334017753,
-      "reward_std": 0.22559863626956939,
-      "rewards/accuracy_reward": 0.14392857477068902,
-      "rewards/format_reward": 0.9603571452200412,
-      "step": 65
-    },
-    {
-      "completion_length": 361.5596435546875,
-      "epoch": 0.10822301671659097,
-      "grad_norm": 0.3746410425327759,
-      "kl": 0.115264892578125,
-      "learning_rate": 9.99817273706943e-06,
-      "loss": 0.0046,
-      "reward": 1.1235714480280876,
-      "reward_std": 0.20448200711980463,
-      "rewards/accuracy_reward": 0.1439285755623132,
-      "rewards/format_reward": 0.9796428583562374,
-      "step": 70
-    },
-    {
-      "completion_length": 348.6992870330811,
-      "epoch": 0.11595323219634747,
-      "grad_norm": 0.3978179778651091,
-      "kl": 0.148199462890625,
-      "learning_rate": 9.992692283833639e-06,
-      "loss": 0.0059,
-      "reward": 1.154285728931427,
-      "reward_std": 0.25590373622253537,
-      "rewards/accuracy_reward": 0.19571429127827286,
-      "rewards/format_reward": 0.9585714310407638,
-      "step": 75
-    },
-    {
-      "completion_length": 360.27392997741697,
-      "epoch": 0.12368344767610397,
-      "grad_norm": 0.3285728265102151,
-      "kl": 0.122698974609375,
-      "learning_rate": 9.983562645984248e-06,
-      "loss": 0.0049,
-      "reward": 1.108214306086302,
-      "reward_std": 0.207218668051064,
-      "rewards/accuracy_reward": 0.1371428606333211,
-      "rewards/format_reward": 0.9710714302957058,
-      "step": 80
-    },
-    {
-      "completion_length": 326.5525007247925,
-      "epoch": 0.13141366315586048,
-      "grad_norm": 0.3701658784343864,
-      "kl": 0.13370361328125,
-      "learning_rate": 9.970790496420781e-06,
-      "loss": 0.0053,
-      "reward": 1.1421428725123406,
-      "reward_std": 0.22562065320089458,
-      "rewards/accuracy_reward": 0.17250000457279385,
-      "rewards/format_reward": 0.9696428589522839,
-      "step": 85
-    },
-    {
-      "completion_length": 340.90464515686034,
-      "epoch": 0.13914387863561697,
-      "grad_norm": 0.3053892797015973,
-      "kl": 0.197650146484375,
-      "learning_rate": 9.954385170373415e-06,
-      "loss": 0.0079,
-      "reward": 1.2007143080234528,
-      "reward_std": 0.223483042512089,
-      "rewards/accuracy_reward": 0.22071429125498981,
-      "rewards/format_reward": 0.9800000011920929,
-      "step": 90
-    },
-    {
-      "completion_length": 370.76178760528563,
-      "epoch": 0.14687409411537347,
-      "grad_norm": 0.26661678402016015,
-      "kl": 0.15225830078125,
-      "learning_rate": 9.934358658579808e-06,
-      "loss": 0.0061,
-      "reward": 1.2128571577370166,
-      "reward_std": 0.24409643467515707,
-      "rewards/accuracy_reward": 0.24714286350645126,
-      "rewards/format_reward": 0.9657142877578735,
-      "step": 95
-    },
-    {
-      "completion_length": 383.44642810821534,
-      "epoch": 0.15460430959512997,
-      "grad_norm": 0.24333486200642604,
-      "kl": 0.140423583984375,
-      "learning_rate": 9.910725598521014e-06,
-      "loss": 0.0056,
-      "reward": 1.1667857356369495,
-      "reward_std": 0.22021743915975095,
-      "rewards/accuracy_reward": 0.19857143445406109,
-      "rewards/format_reward": 0.968214287608862,
-      "step": 100
-    },
-    {
-      "epoch": 0.15460430959512997,
-      "eval_completion_length": 356.63817596435547,
-      "eval_kl": 0.14697265625,
-      "eval_loss": 0.005899494513869286,
-      "eval_reward": 1.2053571343421936,
-      "eval_reward_std": 0.25127829425036907,
-      "eval_rewards/accuracy_reward": 0.23571429401636124,
-      "eval_rewards/format_reward": 0.9696428775787354,
-      "eval_runtime": 85.1811,
-      "eval_samples_per_second": 1.162,
-      "eval_steps_per_second": 0.023,
-      "step": 100
-    },
-    {
-      "completion_length": 381.14428634643554,
-      "epoch": 0.16233452507488647,
-      "grad_norm": 0.32426868486126564,
-      "kl": 0.153460693359375,
-      "learning_rate": 9.883503263722864e-06,
-      "loss": 0.0061,
-      "reward": 1.2335714444518089,
-      "reward_std": 0.23903345353901387,
-      "rewards/accuracy_reward": 0.25892857937142255,
-      "rewards/format_reward": 0.9746428586542606,
-      "step": 105
-    },
-    {
-      "completion_length": 369.8639308929443,
-      "epoch": 0.17006474055464296,
-      "grad_norm": 0.2662792930753043,
-      "kl": 0.16746826171875,
-      "learning_rate": 9.852711551130662e-06,
-      "loss": 0.0067,
-      "reward": 1.2200000181794166,
-      "reward_std": 0.21895755929872393,
-      "rewards/accuracy_reward": 0.24428572261240333,
-      "rewards/format_reward": 0.9757142871618271,
-      "step": 110
-    },
-    {
-      "completion_length": 336.4314300537109,
-      "epoch": 0.17779495603439946,
-      "grad_norm": 0.3050936417250258,
-      "kl": 0.1807861328125,
-      "learning_rate": 9.818372966566404e-06,
-      "loss": 0.0072,
-      "reward": 1.201428586244583,
-      "reward_std": 0.2133496488444507,
-      "rewards/accuracy_reward": 0.22321429231669754,
-      "rewards/format_reward": 0.9782142870128154,
-      "step": 115
-    },
-    {
-      "completion_length": 350.5678596496582,
-      "epoch": 0.18552517151415596,
-      "grad_norm": 0.34458458483928667,
-      "kl": 0.19677734375,
-      "learning_rate": 9.780512608279156e-06,
-      "loss": 0.0079,
-      "reward": 1.146071445941925,
-      "reward_std": 0.2668243307620287,
-      "rewards/accuracy_reward": 0.2042857193853706,
-      "rewards/format_reward": 0.9417857177555561,
-      "step": 120
-    },
-    {
-      "completion_length": 433.7532165527344,
-      "epoch": 0.19325538699391245,
-      "grad_norm": 0.36013965861594144,
-      "kl": 0.220751953125,
-      "learning_rate": 9.739158148600609e-06,
-      "loss": 0.0088,
-      "reward": 1.0971428729593753,
-      "reward_std": 0.3263824795372784,
-      "rewards/accuracy_reward": 0.21000000613275915,
-      "rewards/format_reward": 0.887142863869667,
-      "step": 125
-    },
-    {
-      "completion_length": 497.63500156402586,
-      "epoch": 0.20098560247366895,
-      "grad_norm": 0.34690086218654914,
-      "kl": 0.248583984375,
-      "learning_rate": 9.694339813719243e-06,
-      "loss": 0.0099,
-      "reward": 1.1189285896718502,
-      "reward_std": 0.2914966750890017,
-      "rewards/accuracy_reward": 0.19928571924101562,
-      "rewards/format_reward": 0.9196428619325161,
-      "step": 130
-    },
-    {
-      "completion_length": 326.64821701049806,
-      "epoch": 0.20871581795342545,
-      "grad_norm": 0.33711598936424164,
-      "kl": 0.22991943359375,
-      "learning_rate": 9.646090361587828e-06,
-      "loss": 0.0092,
-      "reward": 1.2050000175833702,
-      "reward_std": 0.23595014372840523,
-      "rewards/accuracy_reward": 0.23321429167408497,
-      "rewards/format_reward": 0.9717857159674168,
-      "step": 135
-    },
-    {
-      "completion_length": 337.5925010681152,
-      "epoch": 0.21644603343318194,
-      "grad_norm": 19.330410175568485,
-      "kl": 0.3224609375,
-      "learning_rate": 9.594445057980483e-06,
-      "loss": 0.0129,
-      "reward": 1.2114285856485367,
-      "reward_std": 0.2494897256605327,
-      "rewards/accuracy_reward": 0.2425000074552372,
-      "rewards/format_reward": 0.9689285732805729,
-      "step": 140
-    },
-    {
-      "completion_length": 359.734645652771,
-      "epoch": 0.22417624891293844,
-      "grad_norm": 1.318993891026559,
-      "kl": 0.32476806640625,
-      "learning_rate": 9.539441650716744e-06,
-      "loss": 0.013,
-      "reward": 1.1532142981886864,
-      "reward_std": 0.268144457321614,
-      "rewards/accuracy_reward": 0.22357143508270383,
-      "rewards/format_reward": 0.9296428613364697,
-      "step": 145
-    },
-    {
-      "completion_length": 308.5442869186401,
-      "epoch": 0.23190646439269494,
-      "grad_norm": 5.642497910156749,
-      "kl": 0.4279296875,
-      "learning_rate": 9.481120342071462e-06,
-      "loss": 0.0171,
-      "reward": 1.1817857310175897,
-      "reward_std": 0.2727494416758418,
-      "rewards/accuracy_reward": 0.23892857865430414,
-      "rewards/format_reward": 0.9428571462631226,
-      "step": 150
-    },
-    {
-      "completion_length": 271.07821559906006,
-      "epoch": 0.23963667987245144,
-      "grad_norm": 12.266848353099874,
-      "kl": 0.36064453125,
-      "learning_rate": 9.41952375939078e-06,
-      "loss": 0.0144,
-      "reward": 1.2067857295274735,
-      "reward_std": 0.2593365713953972,
-      "rewards/accuracy_reward": 0.24535714988596738,
-      "rewards/format_reward": 0.9614285737276077,
-      "step": 155
-    },
-    {
-      "completion_length": 275.08464469909666,
-      "epoch": 0.24736689535220793,
-      "grad_norm": 0.34495859229810133,
-      "kl": 0.226123046875,
-      "learning_rate": 9.354696923935573e-06,
-      "loss": 0.009,
-      "reward": 1.2053571574389934,
-      "reward_std": 0.25177041683346035,
-      "rewards/accuracy_reward": 0.24000000536907465,
-      "rewards/format_reward": 0.9653571449220181,
-      "step": 160
-    },
-    {
-      "completion_length": 250.24928703308106,
-      "epoch": 0.25509711083196446,
-      "grad_norm": 0.3975356570351778,
-      "kl": 0.30623779296875,
-      "learning_rate": 9.28668721797517e-06,
-      "loss": 0.0123,
-      "reward": 1.2103571623563767,
-      "reward_std": 0.2850337457843125,
-      "rewards/accuracy_reward": 0.26000000804197043,
-      "rewards/format_reward": 0.9503571458160878,
-      "step": 165
-    },
-    {
-      "completion_length": 240.8703582763672,
-      "epoch": 0.26282732631172095,
-      "grad_norm": 15.560140664497903,
-      "kl": 0.58157958984375,
-      "learning_rate": 9.215544350155423e-06,
-      "loss": 0.0233,
-      "reward": 1.1975000113248826,
-      "reward_std": 0.23778518568724394,
-      "rewards/accuracy_reward": 0.23892857872415335,
-      "rewards/format_reward": 0.9585714310407638,
-      "step": 170
-    },
-    {
-      "completion_length": 244.13857307434083,
-      "epoch": 0.27055754179147745,
-      "grad_norm": 0.32982217891891835,
-      "kl": 0.33167724609375,
-      "learning_rate": 9.141320319166387e-06,
-      "loss": 0.0133,
-      "reward": 1.2264285899698735,
-      "reward_std": 0.19883861131966113,
-      "rewards/accuracy_reward": 0.24464286372531205,
-      "rewards/format_reward": 0.9817857153713703,
-      "step": 175
-    },
-    {
-      "completion_length": 250.61321525573732,
-      "epoch": 0.27828775727123395,
-      "grad_norm": 0.6994861767616407,
-      "kl": 0.28465576171875,
-      "learning_rate": 9.064069375736216e-06,
-      "loss": 0.0114,
-      "reward": 1.1882142946124077,
-      "reward_std": 0.24957463573664426,
-      "rewards/accuracy_reward": 0.23000000652391464,
-      "rewards/format_reward": 0.9582142882049084,
-      "step": 180
-    },
-    {
-      "completion_length": 172.02142906188965,
-      "epoch": 0.28601797275099045,
-      "grad_norm": 1.042105326460384,
-      "kl": 0.7489013671875,
-      "learning_rate": 8.983847982979022e-06,
-      "loss": 0.03,
-      "reward": 1.0496428698301314,
-      "reward_std": 0.3774342490360141,
-      "rewards/accuracy_reward": 0.18750000472646206,
-      "rewards/format_reward": 0.8621428653597831,
-      "step": 185
-    },
-    {
-      "completion_length": 202.3928581237793,
-      "epoch": 0.29374818823074694,
-      "grad_norm": 0.5558652862204951,
-      "kl": 0.478271484375,
-      "learning_rate": 8.900714775125696e-06,
-      "loss": 0.0191,
-      "reward": 1.119642873853445,
-      "reward_std": 0.2985226408578455,
-      "rewards/accuracy_reward": 0.20392857666593045,
-      "rewards/format_reward": 0.9157142907381057,
-      "step": 190
-    },
-    {
-      "completion_length": 254.10214443206786,
-      "epoch": 0.30147840371050344,
-      "grad_norm": 0.37080340908454873,
-      "kl": 0.543505859375,
-      "learning_rate": 8.814730514667841e-06,
-      "loss": 0.0217,
-      "reward": 1.193214301019907,
-      "reward_std": 0.20587898194789886,
-      "rewards/accuracy_reward": 0.22071429206989707,
-      "rewards/format_reward": 0.9725000016391278,
-      "step": 195
-    },
-    {
-      "completion_length": 239.24893054962158,
-      "epoch": 0.30920861919025994,
-      "grad_norm": 1.419820098954415,
-      "kl": 0.29251708984375,
-      "learning_rate": 8.725958047946155e-06,
-      "loss": 0.0117,
-      "reward": 1.2103571616113187,
-      "reward_std": 0.2034143740311265,
-      "rewards/accuracy_reward": 0.21821429126430303,
-      "rewards/format_reward": 0.9921428576111794,
-      "step": 200
-    },
-    {
-      "epoch": 0.30920861919025994,
-      "eval_completion_length": 248.33312606811523,
-      "eval_kl": 0.275390625,
-      "eval_loss": 0.011179111897945404,
-      "eval_reward": 1.2339285761117935,
-      "eval_reward_std": 0.21089003048837185,
-      "eval_rewards/accuracy_reward": 0.24285714887082577,
-      "eval_rewards/format_reward": 0.9910714328289032,
-      "eval_runtime": 65.1713,
-      "eval_samples_per_second": 1.519,
-      "eval_steps_per_second": 0.031,
-      "step": 200
-    },
-    {
-      "completion_length": 277.3242883682251,
-      "epoch": 0.31693883467001643,
-      "grad_norm": 0.4847949666125137,
-      "kl": 0.28834228515625,
-      "learning_rate": 8.634462259215719e-06,
-      "loss": 0.0115,
-      "reward": 1.224642877280712,
-      "reward_std": 0.21538263475522398,
-      "rewards/accuracy_reward": 0.2310714353341609,
-      "rewards/format_reward": 0.9935714289546013,
-      "step": 205
-    },
-    {
-      "completion_length": 312.77535858154295,
-      "epoch": 0.32466905014977293,
-      "grad_norm": 0.2890010677054343,
-      "kl": 0.29903564453125,
-      "learning_rate": 8.540310023221751e-06,
-      "loss": 0.012,
-      "reward": 1.2089285902678966,
-      "reward_std": 0.21994048971682786,
-      "rewards/accuracy_reward": 0.22392857696395368,
-      "rewards/format_reward": 0.9850000008940697,
-      "step": 210
-    },
-    {
-      "completion_length": 326.0450016021729,
-      "epoch": 0.33239926562952943,
-      "grad_norm": 1.5857581308410953,
-      "kl": 0.3697021484375,
-      "learning_rate": 8.443570156320518e-06,
-      "loss": 0.0148,
-      "reward": 1.2025000154972076,
-      "reward_std": 0.23657457055523992,
-      "rewards/accuracy_reward": 0.22392857796512544,
-      "rewards/format_reward": 0.978571429848671,
-      "step": 215
-    },
-    {
-      "completion_length": 316.9417865753174,
-      "epoch": 0.3401294811092859,
-      "grad_norm": 0.3354478326906057,
-      "kl": 0.335546875,
-      "learning_rate": 8.344313366181096e-06,
-      "loss": 0.0134,
-      "reward": 1.2114285811781884,
-      "reward_std": 0.22043001046404243,
-      "rewards/accuracy_reward": 0.22142857748549433,
-      "rewards/format_reward": 0.9900000005960464,
-      "step": 220
-    },
-    {
-      "completion_length": 322.1989326477051,
-      "epoch": 0.3478596965890424,
-      "grad_norm": 0.3533516561360887,
-      "kl": 0.2961669921875,
-      "learning_rate": 8.242612200104779e-06,
-      "loss": 0.0118,
-      "reward": 1.1957143053412438,
-      "reward_std": 0.1864888435229659,
-      "rewards/accuracy_reward": 0.20000000530853868,
-      "rewards/format_reward": 0.9957142859697342,
-      "step": 225
-    },
-    {
-      "completion_length": 350.877858543396,
-      "epoch": 0.3555899120687989,
-      "grad_norm": 0.2820220582755007,
-      "kl": 0.29697265625,
-      "learning_rate": 8.138540991999871e-06,
-      "loss": 0.0119,
-      "reward": 1.1785714492201804,
-      "reward_std": 0.2050869233906269,
-      "rewards/accuracy_reward": 0.18892857611645014,
-      "rewards/format_reward": 0.989642857760191,
-      "step": 230
-    },
-    {
-      "completion_length": 401.6375032424927,
-      "epoch": 0.3633201275485554,
-      "grad_norm": 3.0020500774885437,
-      "kl": 0.28485107421875,
-      "learning_rate": 8.032175808050659e-06,
-      "loss": 0.0114,
-      "reward": 1.1100000217556953,
-      "reward_std": 0.1967215022072196,
-      "rewards/accuracy_reward": 0.13000000335741788,
-      "rewards/format_reward": 0.9800000011920929,
-      "step": 235
-    },
-    {
-      "completion_length": 355.40107440948486,
-      "epoch": 0.3710503430283119,
-      "grad_norm": 0.28586570118341365,
-      "kl": 0.25643310546875,
-      "learning_rate": 7.923594391120237e-06,
-      "loss": 0.0103,
-      "reward": 1.182142873108387,
-      "reward_std": 0.25666751638054847,
-      "rewards/accuracy_reward": 0.2135714341653511,
-      "rewards/format_reward": 0.9685714304447174,
-      "step": 240
-    },
-    {
-      "completion_length": 343.7985729217529,
-      "epoch": 0.3787805585080684,
-      "grad_norm": 0.31296482492061484,
-      "kl": 0.2564697265625,
-      "learning_rate": 7.812876103927845e-06,
-      "loss": 0.0103,
-      "reward": 1.2296428754925728,
-      "reward_std": 0.24015271971002222,
-      "rewards/accuracy_reward": 0.26000000685453417,
-      "rewards/format_reward": 0.9696428589522839,
-      "step": 245
-    },
-    {
-      "completion_length": 370.3635730743408,
-      "epoch": 0.3865107739878249,
-      "grad_norm": 0.29554233915571987,
-      "kl": 0.308056640625,
-      "learning_rate": 7.700101871042254e-06,
-      "loss": 0.0123,
-      "reward": 1.2103571586310864,
-      "reward_std": 0.24133061729371547,
-      "rewards/accuracy_reward": 0.24000000657979398,
-      "rewards/format_reward": 0.9703571446239948,
-      "step": 250
-    },
-    {
-      "completion_length": 360.6953601837158,
-      "epoch": 0.3942409894675814,
-      "grad_norm": 0.2413456510082092,
-      "kl": 0.24384765625,
-      "learning_rate": 7.5853541197335716e-06,
-      "loss": 0.0098,
-      "reward": 1.2132143005728722,
-      "reward_std": 0.22292996691539885,
-      "rewards/accuracy_reward": 0.23035714915022254,
-      "rewards/format_reward": 0.9828571438789367,
-      "step": 255
-    },
-    {
-      "completion_length": 350.62500114440917,
-      "epoch": 0.4019712049473379,
-      "grad_norm": 0.37205800913968085,
-      "kl": 0.2304443359375,
-      "learning_rate": 7.468716719726733e-06,
-      "loss": 0.0092,
-      "reward": 1.2350000202655793,
-      "reward_std": 0.22238717656582593,
-      "rewards/accuracy_reward": 0.25107143486384303,
-      "rewards/format_reward": 0.9839285723865032,
-      "step": 260
-    },
-    {
-      "completion_length": 345.0939315795898,
-      "epoch": 0.4097014204270944,
-      "grad_norm": 0.5062737177381504,
-      "kl": 0.476416015625,
-      "learning_rate": 7.3502749219006795e-06,
-      "loss": 0.0191,
-      "reward": 1.2007143013179302,
-      "reward_std": 0.24597819074988364,
-      "rewards/accuracy_reward": 0.23928572090808303,
-      "rewards/format_reward": 0.9614285737276077,
-      "step": 265
-    },
-    {
-      "completion_length": 352.0803602218628,
-      "epoch": 0.4174316359068509,
-      "grad_norm": 0.6322459279226639,
-      "kl": 0.49012451171875,
-      "learning_rate": 7.230115295978048e-06,
-      "loss": 0.0196,
-      "reward": 1.1835714481770991,
-      "reward_std": 0.25889406390488146,
-      "rewards/accuracy_reward": 0.23107143563684077,
-      "rewards/format_reward": 0.9525000028312206,
-      "step": 270
-    },
-    {
-      "completion_length": 329.8617877960205,
-      "epoch": 0.4251618513866074,
-      "grad_norm": 0.6668664351542097,
-      "kl": 0.54197998046875,
-      "learning_rate": 7.10832566725092e-06,
-      "loss": 0.0217,
-      "reward": 1.2160714395344256,
-      "reward_std": 0.2636732104234397,
-      "rewards/accuracy_reward": 0.25285714818164706,
-      "rewards/format_reward": 0.9632142879068851,
-      "step": 275
-    },
-    {
-      "completion_length": 332.9325021743774,
-      "epoch": 0.4328920668663639,
-      "grad_norm": 0.862765307967942,
-      "kl": 0.3385986328125,
-      "learning_rate": 6.984995052388856e-06,
-      "loss": 0.0136,
-      "reward": 1.216785727441311,
-      "reward_std": 0.22093856362625958,
-      "rewards/accuracy_reward": 0.2428571493830532,
-      "rewards/format_reward": 0.9739285729825496,
-      "step": 280
-    },
-    {
-      "completion_length": 312.248931312561,
-      "epoch": 0.4406222823461204,
-      "grad_norm": 0.24545660293906987,
-      "kl": 0.2172607421875,
-      "learning_rate": 6.860213594376153e-06,
-      "loss": 0.0087,
-      "reward": 1.2532143011689185,
-      "reward_std": 0.2206911792047322,
-      "rewards/accuracy_reward": 0.2650000074878335,
-      "rewards/format_reward": 0.988214286416769,
-      "step": 285
-    },
-    {
-      "completion_length": 332.121431350708,
-      "epoch": 0.4483524978258769,
-      "grad_norm": 0.26547805533093827,
-      "kl": 0.21544189453125,
-      "learning_rate": 6.734072496625867e-06,
-      "loss": 0.0086,
-      "reward": 1.2425000101327897,
-      "reward_std": 0.22103301845490933,
-      "rewards/accuracy_reward": 0.2614285790361464,
-      "rewards/format_reward": 0.9810714296996593,
-      "step": 290
-    },
-    {
-      "completion_length": 375.60142974853517,
-      "epoch": 0.4560827133056334,
-      "grad_norm": 0.3864015818375869,
-      "kl": 0.29462890625,
-      "learning_rate": 6.606663956318782e-06,
-      "loss": 0.0118,
-      "reward": 1.2335714407265186,
-      "reward_std": 0.2652750977315009,
-      "rewards/accuracy_reward": 0.27428572135977447,
-      "rewards/format_reward": 0.9592857167124749,
-      "step": 295
-    },
-    {
-      "completion_length": 374.28750228881836,
-      "epoch": 0.4638129287853899,
-      "grad_norm": 0.3911583369947814,
-      "kl": 0.4628662109375,
-      "learning_rate": 6.478081097015985e-06,
-      "loss": 0.0185,
-      "reward": 1.2142857298254968,
-      "reward_std": 0.2580754934810102,
-      "rewards/accuracy_reward": 0.26392857884056864,
-      "rewards/format_reward": 0.9503571458160878,
-      "step": 300
-    },
-    {
-      "epoch": 0.4638129287853899,
-      "eval_completion_length": 332.5970458984375,
-      "eval_kl": 0.37158203125,
-      "eval_loss": 0.015323814935982227,
-      "eval_reward": 1.2125000059604645,
-      "eval_reward_std": 0.2901066914200783,
-      "eval_rewards/accuracy_reward": 0.25000000558793545,
-      "eval_rewards/format_reward": 0.9625000134110451,
-      "eval_runtime": 79.9,
-      "eval_samples_per_second": 1.239,
-      "eval_steps_per_second": 0.025,
-      "step": 300
-    },
-    {
-      "completion_length": 343.74143199920655,
-      "epoch": 0.4715431442651464,
-      "grad_norm": 0.38709550861776243,
-      "kl": 0.419091796875,
-      "learning_rate": 6.348417900594408e-06,
-      "loss": 0.0168,
-      "reward": 1.2289285875856877,
-      "reward_std": 0.24545602239668368,
-      "rewards/accuracy_reward": 0.2603571491781622,
-      "rewards/format_reward": 0.9685714304447174,
-      "step": 305
-    },
-    {
-      "completion_length": 334.96392917633057,
-      "epoch": 0.47927335974490287,
-      "grad_norm": 0.4392077094147235,
-      "kl": 0.39736328125,
-      "learning_rate": 6.2177691385549595e-06,
-      "loss": 0.0159,
-      "reward": 1.2221428737044335,
-      "reward_std": 0.21934431809931992,
-      "rewards/accuracy_reward": 0.24250000559259205,
-      "rewards/format_reward": 0.9796428583562374,
-      "step": 310
-    },
-    {
-      "completion_length": 308.76142902374266,
-      "epoch": 0.48700357522465937,
-      "grad_norm": 0.32886450863598027,
-      "kl": 0.333636474609375,
-      "learning_rate": 6.086230302753563e-06,
-      "loss": 0.0133,
-      "reward": 1.2160714440047742,
-      "reward_std": 0.19036251278594135,
-      "rewards/accuracy_reward": 0.23107143498491495,
-      "rewards/format_reward": 0.9850000008940697,
-      "step": 315
-    },
-    {
-      "completion_length": 333.30928802490234,
-      "epoch": 0.49473379070441587,
-      "grad_norm": 1.3946586911147487,
-      "kl": 0.37911376953125,
-      "learning_rate": 5.95389753560565e-06,
-      "loss": 0.0152,
-      "reward": 1.218928585201502,
-      "reward_std": 0.2294909401796758,
-      "rewards/accuracy_reward": 0.24178572059608996,
-      "rewards/format_reward": 0.9771428585052491,
-      "step": 320
-    },
-    {
-      "completion_length": 340.629288482666,
-      "epoch": 0.5024640061841724,
-      "grad_norm": 0.46150339120676787,
-      "kl": 0.36011962890625,
-      "learning_rate": 5.820867559815186e-06,
-      "loss": 0.0144,
-      "reward": 1.236071441322565,
-      "reward_std": 0.21350146597251296,
-      "rewards/accuracy_reward": 0.26500000632368026,
-      "rewards/format_reward": 0.9710714302957058,
-      "step": 325
-    },
-    {
-      "completion_length": 330.6417865753174,
-      "epoch": 0.5101942216639289,
-      "grad_norm": 0.7086286073905499,
-      "kl": 0.37000732421875,
-      "learning_rate": 5.68723760767954e-06,
-      "loss": 0.0148,
-      "reward": 1.2314285844564439,
-      "reward_std": 0.24753115382045507,
-      "rewards/accuracy_reward": 0.263571435585618,
-      "rewards/format_reward": 0.9678571447730064,
-      "step": 330
-    },
-    {
-      "completion_length": 306.0692878723145,
-      "epoch": 0.5179244371436854,
-      "grad_norm": 0.4592159227361041,
-      "kl": 0.43155517578125,
-      "learning_rate": 5.55310535002189e-06,
-      "loss": 0.0173,
-      "reward": 1.2125000201165677,
-      "reward_std": 0.2561903980560601,
-      "rewards/accuracy_reward": 0.24892857796512544,
-      "rewards/format_reward": 0.9635714307427407,
-      "step": 335
-    },
-    {
-      "completion_length": 301.78607177734375,
-      "epoch": 0.5256546526234419,
-      "grad_norm": 1.3334554805750072,
-      "kl": 0.40533447265625,
-      "learning_rate": 5.418568824803121e-06,
-      "loss": 0.0162,
-      "reward": 1.2046428814530372,
-      "reward_std": 0.24458126006647946,
-      "rewards/accuracy_reward": 0.23857143591158092,
-      "rewards/format_reward": 0.966071430593729,
-      "step": 340
-    },
-    {
-      "completion_length": 311.9282154083252,
-      "epoch": 0.5333848681031984,
-      "grad_norm": 0.49010519701579414,
-      "kl": 0.47899169921875,
-      "learning_rate": 5.2837263654653715e-06,
-      "loss": 0.0192,
-      "reward": 1.2089285902678966,
-      "reward_std": 0.24385176142677664,
-      "rewards/accuracy_reward": 0.24214286329224705,
-      "rewards/format_reward": 0.96678571626544,
-      "step": 345
-    },
-    {
-      "completion_length": 356.58785762786863,
-      "epoch": 0.5411150835829549,
-      "grad_norm": 0.40401739909507167,
-      "kl": 0.297998046875,
-      "learning_rate": 5.148676529059604e-06,
-      "loss": 0.0119,
-      "reward": 1.2339285902678967,
-      "reward_std": 0.2254973877221346,
-      "rewards/accuracy_reward": 0.25178572128061205,
-      "rewards/format_reward": 0.9821428582072258,
-      "step": 350
-    },
-    {
-      "completion_length": 401.15607299804685,
-      "epoch": 0.5488452990627114,
-      "grad_norm": 0.43919579917421436,
-      "kl": 0.467236328125,
-      "learning_rate": 5.013518024209757e-06,
-      "loss": 0.0187,
-      "reward": 1.223928590863943,
-      "reward_std": 0.25751951709389687,
-      "rewards/accuracy_reward": 0.2525000067660585,
-      "rewards/format_reward": 0.9714285731315613,
-      "step": 355
-    },
-    {
-      "completion_length": 372.611074256897,
-      "epoch": 0.5565755145424679,
-      "grad_norm": 0.5922777261314072,
-      "kl": 0.43697509765625,
-      "learning_rate": 4.878349638966098e-06,
-      "loss": 0.0175,
-      "reward": 1.2207143008708954,
-      "reward_std": 0.2579284094274044,
-      "rewards/accuracy_reward": 0.25178572118747977,
-      "rewards/format_reward": 0.9689285732805729,
-      "step": 360
-    },
-    {
-      "completion_length": 351.6828565597534,
-      "epoch": 0.5643057300222244,
-      "grad_norm": 0.29650599886416334,
-      "kl": 0.3662109375,
-      "learning_rate": 4.743270168600523e-06,
-      "loss": 0.0146,
-      "reward": 1.2132143080234528,
-      "reward_std": 0.21272512217983602,
-      "rewards/accuracy_reward": 0.23607143440749495,
-      "rewards/format_reward": 0.9771428585052491,
-      "step": 365
-    },
-    {
-      "completion_length": 352.63428859710694,
-      "epoch": 0.5720359455019809,
-      "grad_norm": 0.2713235178095771,
-      "kl": 0.32996826171875,
-      "learning_rate": 4.608378343396581e-06,
-      "loss": 0.0132,
-      "reward": 1.2214285895228385,
-      "reward_std": 0.2412548642605543,
-      "rewards/accuracy_reward": 0.24821429257281125,
-      "rewards/format_reward": 0.9732142873108387,
-      "step": 370
-    },
-    {
-      "completion_length": 400.2235725402832,
-      "epoch": 0.5797661609817374,
-      "grad_norm": 0.6530373844158506,
-      "kl": 0.534814453125,
-      "learning_rate": 4.47377275648701e-06,
-      "loss": 0.0214,
-      "reward": 1.1946428716182709,
-      "reward_std": 0.2843459703028202,
-      "rewards/accuracy_reward": 0.24785715015605092,
-      "rewards/format_reward": 0.9467857174575329,
-      "step": 375
-    },
-    {
-      "completion_length": 384.1235717773437,
-      "epoch": 0.5874963764614939,
-      "grad_norm": 0.5620739725200156,
-      "kl": 0.433917236328125,
-      "learning_rate": 4.33955179179149e-06,
-      "loss": 0.0174,
-      "reward": 1.1839285880327224,
-      "reward_std": 0.27014439897611736,
-      "rewards/accuracy_reward": 0.2300000066170469,
-      "rewards/format_reward": 0.9539285741746426,
-      "step": 380
-    },
-    {
-      "completion_length": 367.36893062591554,
-      "epoch": 0.5952265919412504,
-      "grad_norm": 0.3417788309196838,
-      "kl": 0.45970458984375,
-      "learning_rate": 4.205813552107337e-06,
-      "loss": 0.0184,
-      "reward": 1.2242857284843922,
-      "reward_std": 0.2591902747750282,
-      "rewards/accuracy_reward": 0.26214286310132595,
-      "rewards/format_reward": 0.9621428593993187,
-      "step": 385
-    },
-    {
-      "completion_length": 344.9664316177368,
-      "epoch": 0.6029568074210069,
-      "grad_norm": 0.383725508536428,
-      "kl": 0.2947509765625,
-      "learning_rate": 4.0726557874056625e-06,
-      "loss": 0.0118,
-      "reward": 1.1982143066823483,
-      "reward_std": 0.21319182245060803,
-      "rewards/accuracy_reward": 0.21928572067990898,
-      "rewards/format_reward": 0.9789285726845265,
-      "step": 390
-    },
-    {
-      "completion_length": 325.8582160949707,
-      "epoch": 0.6106870229007634,
-      "grad_norm": 0.5862839492831149,
-      "kl": 0.36025390625,
-      "learning_rate": 3.940175823385412e-06,
-      "loss": 0.0144,
-      "reward": 1.2221428655087947,
-      "reward_std": 0.24327749805524945,
-      "rewards/accuracy_reward": 0.24928572128992527,
-      "rewards/format_reward": 0.9728571444749832,
-      "step": 395
-    },
-    {
-      "completion_length": 347.8882158279419,
-      "epoch": 0.6184172383805199,
-      "grad_norm": 0.47993598719208436,
-      "kl": 0.2891845703125,
-      "learning_rate": 3.808470490337502e-06,
-      "loss": 0.0116,
-      "reward": 1.2317857250571251,
-      "reward_std": 0.2260462765581906,
-      "rewards/accuracy_reward": 0.2532142927404493,
-      "rewards/format_reward": 0.978571429848671,
-      "step": 400
-    },
-    {
-      "epoch": 0.6184172383805199,
-      "eval_completion_length": 318.71398162841797,
-      "eval_kl": 0.3338623046875,
-      "eval_loss": 0.013500670902431011,
-      "eval_reward": 1.2374999970197678,
-      "eval_reward_std": 0.24992241524159908,
-      "eval_rewards/accuracy_reward": 0.2607142888009548,
-      "eval_rewards/format_reward": 0.976785734295845,
-      "eval_runtime": 77.5724,
-      "eval_samples_per_second": 1.276,
-      "eval_steps_per_second": 0.026,
-      "step": 400
-    },
-    {
-      "completion_length": 340.0689334869385,
-      "epoch": 0.6261474538602764,
-      "grad_norm": 0.5074418639717388,
-      "kl": 0.3012939453125,
-      "learning_rate": 3.677636052371066e-06,
-      "loss": 0.0121,
-      "reward": 1.2257142968475818,
-      "reward_std": 0.23705143565312028,
-      "rewards/accuracy_reward": 0.2507142927730456,
-      "rewards/format_reward": 0.9750000014901161,
-      "step": 405
-    },
-    {
-      "completion_length": 334.34285869598386,
-      "epoch": 0.6338776693400329,
-      "grad_norm": 0.440989596288644,
-      "kl": 0.5120361328125,
-      "learning_rate": 3.547768137053523e-06,
-      "loss": 0.0205,
-      "reward": 1.2110714435577392,
-      "reward_std": 0.24922448974102734,
-      "rewards/accuracy_reward": 0.24642857799772172,
-      "rewards/format_reward": 0.9646428592503071,
-      "step": 410
-    },
-    {
-      "completion_length": 330.91714363098146,
-      "epoch": 0.6416078848197894,
-      "grad_norm": 0.26694024078543577,
-      "kl": 0.28255615234375,
-      "learning_rate": 3.4189616655158803e-06,
-      "loss": 0.0113,
-      "reward": 1.2403571613132953,
-      "reward_std": 0.2156613773666322,
-      "rewards/accuracy_reward": 0.2560714353807271,
-      "rewards/format_reward": 0.9842857152223587,
-      "step": 415
-    },
-    {
-      "completion_length": 373.41714534759524,
-      "epoch": 0.6493381002995459,
-      "grad_norm": 0.32331429774400394,
-      "kl": 0.262451171875,
-      "learning_rate": 3.291310783074405e-06,
-      "loss": 0.0105,
-      "reward": 1.26178572550416,
-      "reward_std": 0.22110692989081143,
-      "rewards/accuracy_reward": 0.2775000070221722,
-      "rewards/format_reward": 0.9842857152223587,
-      "step": 420
-    },
-    {
-      "completion_length": 410.3825017929077,
-      "epoch": 0.6570683157793024,
-      "grad_norm": 0.36969945354312556,
-      "kl": 0.34056396484375,
-      "learning_rate": 3.1649087904193142e-06,
-      "loss": 0.0136,
-      "reward": 1.2367857284843922,
-      "reward_std": 0.2502147083170712,
-      "rewards/accuracy_reward": 0.2603571502259001,
-      "rewards/format_reward": 0.976428572833538,
-      "step": 425
-    },
-    {
-      "completion_length": 401.2521469116211,
-      "epoch": 0.6647985312590589,
-      "grad_norm": 0.6482466478004806,
-      "kl": 0.38873291015625,
-      "learning_rate": 3.039848075420817e-06,
-      "loss": 0.0156,
-      "reward": 1.2332143016159534,
-      "reward_std": 0.2610732516273856,
-      "rewards/accuracy_reward": 0.2721428648335859,
-      "rewards/format_reward": 0.9610714308917523,
-      "step": 430
-    },
-    {
-      "completion_length": 416.10785961151123,
-      "epoch": 0.6725287467388154,
-      "grad_norm": 1.0368723402917392,
-      "kl": 0.44521484375,
-      "learning_rate": 2.916220045602349e-06,
-      "loss": 0.0178,
-      "reward": 1.2125000186264514,
-      "reward_std": 0.2809831270948052,
-      "rewards/accuracy_reward": 0.25500000710599124,
-      "rewards/format_reward": 0.9575000025331974,
-      "step": 435
-    },
-    {
-      "completion_length": 396.11535835266113,
-      "epoch": 0.6802589622185718,
-      "grad_norm": 0.4245630889591221,
-      "kl": 0.42137451171875,
-      "learning_rate": 2.794115061330336e-06,
-      "loss": 0.0169,
-      "reward": 1.2089285865426063,
-      "reward_std": 0.27289886865764856,
-      "rewards/accuracy_reward": 0.25464286408387127,
-      "rewards/format_reward": 0.954285717010498,
-      "step": 440
-    },
-    {
-      "completion_length": 405.51571731567384,
-      "epoch": 0.6879891776983283,
-      "grad_norm": 0.47571162730800487,
-      "kl": 0.45782470703125,
-      "learning_rate": 2.6736223697693387e-06,
-      "loss": 0.0183,
-      "reward": 1.21464286968112,
-      "reward_std": 0.2741479127667844,
-      "rewards/accuracy_reward": 0.2639285788871348,
-      "rewards/format_reward": 0.9507142886519432,
-      "step": 445
-    },
-    {
-      "completion_length": 378.1635715484619,
-      "epoch": 0.6957193931780848,
-      "grad_norm": 0.43896790616629155,
-      "kl": 0.4071044921875,
-      "learning_rate": 2.554830039650834e-06,
-      "loss": 0.0163,
-      "reward": 1.202500019222498,
-      "reward_std": 0.2554827404208481,
-      "rewards/accuracy_reward": 0.24857143468689175,
-      "rewards/format_reward": 0.9539285741746426,
-      "step": 450
-    },
-    {
-      "completion_length": 376.27357387542725,
-      "epoch": 0.7034496086578413,
-      "grad_norm": 0.6375781082839199,
-      "kl": 0.40799560546875,
-      "learning_rate": 2.4378248969033275e-06,
-      "loss": 0.0163,
-      "reward": 1.2100000150501729,
-      "reward_std": 0.2629528701305389,
-      "rewards/accuracy_reward": 0.24857143517583608,
-      "rewards/format_reward": 0.9614285737276077,
-      "step": 455
-    },
-    {
-      "completion_length": 400.56857414245604,
-      "epoch": 0.7111798241375978,
-      "grad_norm": 0.5893322722521851,
-      "kl": 0.46832275390625,
-      "learning_rate": 2.32269246119083e-06,
-      "loss": 0.0187,
-      "reward": 1.1839285910129547,
-      "reward_std": 0.2871158769354224,
-      "rewards/accuracy_reward": 0.24214286401402205,
-      "rewards/format_reward": 0.9417857177555561,
-      "step": 460
-    },
-    {
-      "completion_length": 426.16643199920657,
-      "epoch": 0.7189100396173543,
-      "grad_norm": 0.7285531480050851,
-      "kl": 0.45289306640625,
-      "learning_rate": 2.2095168834060982e-06,
-      "loss": 0.0181,
-      "reward": 1.213928584754467,
-      "reward_std": 0.27019564192742107,
-      "rewards/accuracy_reward": 0.2635714357485995,
-      "rewards/format_reward": 0.9503571458160878,
-      "step": 465
-    },
-    {
-      "completion_length": 426.24428634643556,
-      "epoch": 0.7266402550971108,
-      "grad_norm": 0.3644894126467112,
-      "kl": 0.44105224609375,
-      "learning_rate": 2.0983808841642996e-06,
-      "loss": 0.0176,
-      "reward": 1.227142871916294,
-      "reward_std": 0.2584091882221401,
-      "rewards/accuracy_reward": 0.2635714358650148,
-      "rewards/format_reward": 0.9635714307427407,
-      "step": 470
-    },
-    {
-      "completion_length": 398.2003578186035,
-      "epoch": 0.7343704705768673,
-      "grad_norm": 0.40238625933610955,
-      "kl": 0.339495849609375,
-      "learning_rate": 1.989365693342103e-06,
-      "loss": 0.0136,
-      "reward": 1.2378571562469005,
-      "reward_std": 0.24942016880959272,
-      "rewards/accuracy_reward": 0.2696428646799177,
-      "rewards/format_reward": 0.968214287608862,
-      "step": 475
-    },
-    {
-      "completion_length": 417.70250301361085,
-      "epoch": 0.7421006860566238,
-      "grad_norm": 0.4343672927770361,
-      "kl": 0.293597412109375,
-      "learning_rate": 1.8825509907063328e-06,
-      "loss": 0.0117,
-      "reward": 1.245714296400547,
-      "reward_std": 0.23920459682121872,
-      "rewards/accuracy_reward": 0.2710714365821332,
-      "rewards/format_reward": 0.9746428586542606,
-      "step": 480
-    },
-    {
-      "completion_length": 402.52250442504885,
-      "epoch": 0.7498309015363803,
-      "grad_norm": 0.5328436426470834,
-      "kl": 0.333837890625,
-      "learning_rate": 1.7780148476756148e-06,
-      "loss": 0.0134,
-      "reward": 1.2360714450478554,
-      "reward_std": 0.2392024408094585,
-      "rewards/accuracy_reward": 0.2632142927031964,
-      "rewards/format_reward": 0.9728571444749832,
-      "step": 485
-    },
-    {
-      "completion_length": 418.83428859710693,
-      "epoch": 0.7575611170161368,
-      "grad_norm": 0.3887859392004243,
-      "kl": 0.3875732421875,
-      "learning_rate": 1.6758336702575773e-06,
-      "loss": 0.0155,
-      "reward": 1.2235714450478554,
-      "reward_std": 0.26047459337860346,
-      "rewards/accuracy_reward": 0.25464286440983414,
-      "rewards/format_reward": 0.9689285732805729,
-      "step": 490
-    },
-    {
-      "completion_length": 418.6089317321777,
-      "epoch": 0.7652913324958933,
-      "grad_norm": 0.5048781836979007,
-      "kl": 0.3671630859375,
-      "learning_rate": 1.5760821432032953e-06,
-      "loss": 0.0147,
-      "reward": 1.2132142998278141,
-      "reward_std": 0.24158032452687622,
-      "rewards/accuracy_reward": 0.24642857771832496,
-      "rewards/format_reward": 0.96678571626544,
-      "step": 495
-    },
-    {
-      "completion_length": 401.3010726928711,
-      "epoch": 0.7730215479756498,
-      "grad_norm": 0.4607265236386812,
-      "kl": 0.4398681640625,
-      "learning_rate": 1.4788331754198316e-06,
-      "loss": 0.0176,
-      "reward": 1.2085714474320413,
-      "reward_std": 0.28208895614370705,
-      "rewards/accuracy_reward": 0.2539285784587264,
-      "rewards/format_reward": 0.9546428598463536,
-      "step": 500
-    },
-    {
-      "epoch": 0.7730215479756498,
-      "eval_completion_length": 354.90320205688477,
-      "eval_kl": 0.45703125,
-      "eval_loss": 0.018093252554535866,
-      "eval_reward": 1.226785734295845,
-      "eval_reward_std": 0.26129008270800114,
-      "eval_rewards/accuracy_reward": 0.2696428634226322,
-      "eval_rewards/format_reward": 0.9571428745985031,
-      "eval_runtime": 110.1034,
-      "eval_samples_per_second": 0.899,
-      "eval_steps_per_second": 0.018,
-      "step": 500
-    },
-    {
-      "completion_length": 402.9871459960938,
-      "epoch": 0.7807517634554063,
-      "grad_norm": 0.520985901336068,
-      "kl": 0.4232177734375,
-      "learning_rate": 1.384157846680737e-06,
-      "loss": 0.0169,
-      "reward": 1.2203571587800979,
-      "reward_std": 0.24645365020260215,
-      "rewards/accuracy_reward": 0.2632142924237996,
-      "rewards/format_reward": 0.9571428596973419,
-      "step": 505
-    },
-    {
-      "completion_length": 407.93928661346433,
-      "epoch": 0.7884819789351628,
-      "grad_norm": 0.3805648616018147,
-      "kl": 0.384832763671875,
-      "learning_rate": 1.2921253556734654e-06,
-      "loss": 0.0154,
-      "reward": 1.217857153713703,
-      "reward_std": 0.26537872953340413,
-      "rewards/accuracy_reward": 0.25535714966244993,
-      "rewards/format_reward": 0.9625000022351742,
-      "step": 510
-    },
-    {
-      "completion_length": 388.97393188476565,
-      "epoch": 0.7962121944149193,
-      "grad_norm": 1.1499390375989664,
-      "kl": 0.318841552734375,
-      "learning_rate": 1.202802969421708e-06,
-      "loss": 0.0128,
-      "reward": 1.198214303702116,
-      "reward_std": 0.23692603250965477,
-      "rewards/accuracy_reward": 0.23464286390226335,
-      "rewards/format_reward": 0.9635714307427407,
-      "step": 515
-    },
-    {
-      "completion_length": 383.35643138885496,
-      "epoch": 0.8039424098946758,
-      "grad_norm": 0.4367989993018896,
-      "kl": 0.34908447265625,
-      "learning_rate": 1.1162559741195733e-06,
-      "loss": 0.014,
-      "reward": 1.245357161015272,
-      "reward_std": 0.2558293900452554,
-      "rewards/accuracy_reward": 0.27678572088479997,
-      "rewards/format_reward": 0.9685714304447174,
-      "step": 520
-    },
-    {
-      "completion_length": 385.04857444763184,
-      "epoch": 0.8116726253744323,
-      "grad_norm": 0.6398289536823912,
-      "kl": 0.329840087890625,
-      "learning_rate": 1.0325476274135665e-06,
-      "loss": 0.0132,
-      "reward": 1.2332143023610116,
-      "reward_std": 0.23316460549831391,
-      "rewards/accuracy_reward": 0.2617857217788696,
-      "rewards/format_reward": 0.9714285731315613,
-      "step": 525
-    },
-    {
-      "completion_length": 385.87678928375243,
-      "epoch": 0.8194028408541888,
-      "grad_norm": 0.3695626628673057,
-      "kl": 0.39681396484375,
-      "learning_rate": 9.517391121672537e-07,
-      "loss": 0.0159,
-      "reward": 1.2292857207357883,
-      "reward_std": 0.2500005507841706,
-      "rewards/accuracy_reward": 0.26821429268457,
-      "rewards/format_reward": 0.9610714308917523,
-      "step": 530
-    },
-    {
-      "completion_length": 370.4653594970703,
-      "epoch": 0.8271330563339453,
-      "grad_norm": 0.6854915615610104,
-      "kl": 0.38438720703125,
-      "learning_rate": 8.738894917423918e-07,
-      "loss": 0.0154,
-      "reward": 1.2175000138580798,
-      "reward_std": 0.24949982650578023,
-      "rewards/accuracy_reward": 0.2521428640931845,
-      "rewards/format_reward": 0.9653571449220181,
-      "step": 535
-    },
-    {
-      "completion_length": 373.39250297546386,
-      "epoch": 0.8348632718137018,
-      "grad_norm": 0.3428641792816048,
-      "kl": 0.3291259765625,
-      "learning_rate": 7.990556668292021e-07,
-      "loss": 0.0132,
-      "reward": 1.2228571593761444,
-      "reward_std": 0.2329377906396985,
-      "rewards/accuracy_reward": 0.2507142930058762,
-      "rewards/format_reward": 0.9721428588032722,
-      "step": 540
-    },
-    {
-      "completion_length": 390.6789312362671,
-      "epoch": 0.8425934872934583,
-      "grad_norm": 0.8781910108571961,
-      "kl": 0.3635009765625,
-      "learning_rate": 7.272923338573723e-07,
-      "loss": 0.0145,
-      "reward": 1.2246428735554218,
-      "reward_std": 0.24744346737861633,
-      "rewards/accuracy_reward": 0.2567857220768929,
-      "rewards/format_reward": 0.9678571447730064,
-      "step": 545
-    },
-    {
-      "completion_length": 386.65393047332765,
-      "epoch": 0.8503237027732148,
-      "grad_norm": 0.374646921874402,
-      "kl": 0.305267333984375,
-      "learning_rate": 6.586519450181488e-07,
-      "loss": 0.0122,
-      "reward": 1.242500015348196,
-      "reward_std": 0.24353330470621587,
-      "rewards/accuracy_reward": 0.26964286335278304,
-      "rewards/format_reward": 0.9728571444749832,
-      "step": 550
-    },
-    {
-      "completion_length": 384.5810733795166,
-      "epoch": 0.8580539182529713,
-      "grad_norm": 0.5549528232027495,
-      "kl": 0.312255859375,
-      "learning_rate": 5.931846699267558e-07,
-      "loss": 0.0125,
-      "reward": 1.247857154905796,
-      "reward_std": 0.25878492267802355,
-      "rewards/accuracy_reward": 0.2767857232829556,
-      "rewards/format_reward": 0.9710714302957058,
-      "step": 555
-    },
-    {
-      "completion_length": 386.3992879867554,
-      "epoch": 0.8657841337327278,
-      "grad_norm": 0.4278411326159335,
-      "kl": 0.34853515625,
-      "learning_rate": 5.309383589531708e-07,
-      "loss": 0.0139,
-      "reward": 1.2271428763866425,
-      "reward_std": 0.23170938733965157,
-      "rewards/accuracy_reward": 0.2575000062817708,
-      "rewards/format_reward": 0.9696428589522839,
-      "step": 560
-    },
-    {
-      "completion_length": 391.38536109924314,
-      "epoch": 0.8735143492124843,
-      "grad_norm": 0.4003970163311169,
-      "kl": 0.37874755859375,
-      "learning_rate": 4.7195850824803823e-07,
-      "loss": 0.0152,
-      "reward": 1.2192857332527638,
-      "reward_std": 0.25189957171678545,
-      "rewards/accuracy_reward": 0.2528571498580277,
-      "rewards/format_reward": 0.9664285734295845,
-      "step": 565
-    },
-    {
-      "completion_length": 384.1317882537842,
-      "epoch": 0.8812445646922408,
-      "grad_norm": 0.4111180386794067,
-      "kl": 0.358599853515625,
-      "learning_rate": 4.1628822648929255e-07,
-      "loss": 0.0143,
-      "reward": 1.2210714399814606,
-      "reward_std": 0.24504473190754653,
-      "rewards/accuracy_reward": 0.25392857876140623,
-      "rewards/format_reward": 0.9671428591012955,
-      "step": 570
-    },
-    {
-      "completion_length": 400.0792875289917,
-      "epoch": 0.8889747801719973,
-      "grad_norm": 0.44019676078400016,
-      "kl": 0.317877197265625,
-      "learning_rate": 3.6396820337381236e-07,
-      "loss": 0.0127,
-      "reward": 1.2107143074274063,
-      "reward_std": 0.23559978352859617,
-      "rewards/accuracy_reward": 0.24178572117816657,
-      "rewards/format_reward": 0.9689285732805729,
-      "step": 575
-    },
-    {
-      "completion_length": 391.342502784729,
-      "epoch": 0.8967049956517538,
-      "grad_norm": 0.46727020273244035,
-      "kl": 0.312957763671875,
-      "learning_rate": 3.150366798771043e-07,
-      "loss": 0.0125,
-      "reward": 1.2439285896718502,
-      "reward_std": 0.2462494120001793,
-      "rewards/accuracy_reward": 0.2714285790920258,
-      "rewards/format_reward": 0.9725000016391278,
-      "step": 580
-    },
-    {
-      "completion_length": 372.30536060333253,
-      "epoch": 0.9044352111315103,
-      "grad_norm": 0.5209042194427927,
-      "kl": 0.319873046875,
-      "learning_rate": 2.6952942030277486e-07,
-      "loss": 0.0128,
-      "reward": 1.2671428687870503,
-      "reward_std": 0.26302206302061676,
-      "rewards/accuracy_reward": 0.2996428650803864,
-      "rewards/format_reward": 0.9675000019371509,
-      "step": 585
-    },
-    {
-      "completion_length": 394.72285861968993,
-      "epoch": 0.9121654266112668,
-      "grad_norm": 0.4137939679586483,
-      "kl": 0.33170166015625,
-      "learning_rate": 2.274796861422246e-07,
-      "loss": 0.0133,
-      "reward": 1.2225000075995922,
-      "reward_std": 0.24834517342969775,
-      "rewards/accuracy_reward": 0.25821429330389944,
-      "rewards/format_reward": 0.9642857164144516,
-      "step": 590
-    },
-    {
-      "completion_length": 402.35428485870364,
-      "epoch": 0.9198956420910233,
-      "grad_norm": 0.412787643650894,
-      "kl": 0.37198486328125,
-      "learning_rate": 1.88918211763639e-07,
-      "loss": 0.0149,
-      "reward": 1.2421428732573987,
-      "reward_std": 0.2519141098484397,
-      "rewards/accuracy_reward": 0.2735714362002909,
-      "rewards/format_reward": 0.9685714304447174,
-      "step": 595
-    },
-    {
-      "completion_length": 381.8421464920044,
-      "epoch": 0.9276258575707798,
-      "grad_norm": 0.4865523678871763,
-      "kl": 0.323974609375,
-      "learning_rate": 1.5387318194809098e-07,
-      "loss": 0.013,
-      "reward": 1.2353571593761443,
-      "reward_std": 0.22969860946759582,
-      "rewards/accuracy_reward": 0.2639285792130977,
-      "rewards/format_reward": 0.9714285731315613,
-      "step": 600
-    },
-    {
-      "epoch": 0.9276258575707798,
-      "eval_completion_length": 353.1224174499512,
-      "eval_kl": 0.30078125,
-      "eval_loss": 0.011807209812104702,
-      "eval_reward": 1.2553571313619614,
-      "eval_reward_std": 0.22612838447093964,
-      "eval_rewards/accuracy_reward": 0.2821428645402193,
-      "eval_rewards/format_reward": 0.973214291036129,
-      "eval_runtime": 111.1459,
-      "eval_samples_per_second": 0.891,
-      "eval_steps_per_second": 0.018,
-      "step": 600
-    },
-    {
-      "completion_length": 404.2853603363037,
-      "epoch": 0.9353560730505363,
-      "grad_norm": 0.5171782261479181,
-      "kl": 0.34097900390625,
-      "learning_rate": 1.223702112891345e-07,
-      "loss": 0.0136,
-      "reward": 1.2089285850524902,
-      "reward_std": 0.24284306410700082,
-      "rewards/accuracy_reward": 0.2428571501513943,
-      "rewards/format_reward": 0.966071430593729,
-      "step": 605
-    },
-    {
-      "completion_length": 397.3825029373169,
-      "epoch": 0.9430862885302927,
-      "grad_norm": 0.7320475164079379,
-      "kl": 0.3185546875,
-      "learning_rate": 9.443232547096448e-08,
-      "loss": 0.0127,
-      "reward": 1.256428588926792,
-      "reward_std": 0.26404928769916297,
-      "rewards/accuracy_reward": 0.2846428649034351,
-      "rewards/format_reward": 0.9717857159674168,
-      "step": 610
-    },
-    {
-      "completion_length": 407.19285888671874,
-      "epoch": 0.9508165040100492,
-      "grad_norm": 1.2171362297470418,
-      "kl": 0.36600341796875,
-      "learning_rate": 7.007994443882616e-08,
-      "loss": 0.0146,
-      "reward": 1.210714304447174,
-      "reward_std": 0.26716623855754734,
-      "rewards/accuracy_reward": 0.24892857847735286,
-      "rewards/format_reward": 0.9617857165634632,
-      "step": 615
-    },
-    {
-      "completion_length": 395.53178749084475,
-      "epoch": 0.9585467194898057,
-      "grad_norm": 0.36444035142958237,
-      "kl": 0.3221923828125,
-      "learning_rate": 4.933086747397231e-08,
-      "loss": 0.0129,
-      "reward": 1.2253571592271328,
-      "reward_std": 0.23916244860738517,
-      "rewards/accuracy_reward": 0.2553571488009766,
-      "rewards/format_reward": 0.9700000017881394,
-      "step": 620
-    },
-    {
-      "completion_length": 393.6335744857788,
-      "epoch": 0.9662769349695622,
-      "grad_norm": 0.3989424097774144,
-      "kl": 0.3381103515625,
-      "learning_rate": 3.220026018407541e-08,
-      "loss": 0.0135,
-      "reward": 1.2310714446008206,
-      "reward_std": 0.23451657630503178,
-      "rewards/accuracy_reward": 0.2621428639627993,
-      "rewards/format_reward": 0.9689285732805729,
-      "step": 625
-    },
-    {
-      "completion_length": 396.7300016403198,
-      "epoch": 0.9740071504493187,
-      "grad_norm": 0.7711804308584844,
-      "kl": 0.33509521484375,
-      "learning_rate": 1.8700643418607112e-08,
-      "loss": 0.0134,
-      "reward": 1.250000011920929,
-      "reward_std": 0.26608177088201046,
-      "rewards/accuracy_reward": 0.2835714362096041,
-      "rewards/format_reward": 0.9664285734295845,
-      "step": 630
-    },
-    {
-      "completion_length": 392.350359916687,
-      "epoch": 0.9817373659290752,
-      "grad_norm": 0.42013697611704715,
-      "kl": 0.327276611328125,
-      "learning_rate": 8.841884117284372e-09,
-      "loss": 0.0131,
-      "reward": 1.241785728931427,
-      "reward_std": 0.24906472209841013,
-      "rewards/accuracy_reward": 0.270714293140918,
-      "rewards/format_reward": 0.9710714302957058,
-      "step": 635
-    },
-    {
-      "completion_length": 379.29678897857667,
-      "epoch": 0.9894675814088317,
-      "grad_norm": 0.39572354868452714,
-      "kl": 0.298919677734375,
-      "learning_rate": 2.6311880982721195e-09,
-      "loss": 0.012,
-      "reward": 1.248571439087391,
-      "reward_std": 0.25294895516708493,
-      "rewards/accuracy_reward": 0.2753571497276425,
-      "rewards/format_reward": 0.9732142873108387,
-      "step": 640
-    },
-    {
-      "completion_length": 391.0450023651123,
-      "epoch": 0.9971977968885882,
-      "grad_norm": 0.4471955331682869,
-      "kl": 0.334100341796875,
-      "learning_rate": 7.30947914140412e-11,
-      "loss": 0.0134,
-      "reward": 1.2375000178813935,
-      "reward_std": 0.24879253581166266,
-      "rewards/accuracy_reward": 0.26464286535046994,
-      "rewards/format_reward": 0.9728571444749832,
-      "step": 645
-    },
-    {
-      "completion_length": 370.5642910003662,
-      "epoch": 0.9987438399845395,
-      "kl": 0.784271240234375,
-      "reward": 1.2250000201165676,
-      "reward_std": 0.259319587610662,
-      "rewards/accuracy_reward": 0.25714286393485963,
-      "rewards/format_reward": 0.9678571447730064,
-      "step": 646,
+      "completion_length": 462.55078125,
+      "epoch": 0,
+      "kl": 0.0,
+      "reward": 0.453125,
+      "reward_std": 0.46982569620013237,
+      "rewards/accuracy_reward": 0.08984375,
+      "rewards/format_reward": 0.36328125,
+      "step": 0,
       "total_flos": 0.0,
-      "train_loss": 0.01271514138510042,
-      "train_runtime": 147461.7668,
-      "train_samples_per_second": 0.491,
-      "train_steps_per_second": 0.004
+      "train_loss": -42.93539375066757,
+      "train_runtime": 225.8365,
+      "train_samples_per_second": 0.177,
+      "train_steps_per_second": 0.009
     }
   ],
   "logging_steps": 5,
-  "max_steps": 646,
+  "max_steps": 2,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 2,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -1803,7 +42,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 5,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcb5bdda4f17eb1fd0d6349eced4d9ec471f751107ad5bf83f361ddac1e00393
+oid sha256:206e34d72e4e2f1295217d7e9bea8d091ac026393e1a945faa65caed9904ae9a
 size 7544