Model save

README.md

@@ -1,11 +1,9 @@
 ---
-base_model: Qwen/Qwen3-1.7B
-datasets: Blancy/verifiable-coding-problems-CoT
+base_model: Qwen/Qwen3-0.6B
 library_name: transformers
 model_name: Qwen3-1.7B-Open-R1-Code-GRPO
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen3-1.7B-Open-R1-Code-GRPO
 
-This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B) on the [Blancy/verifiable-coding-problems-CoT](https://huggingface.co/datasets/Blancy/verifiable-coding-problems-CoT) dataset.
+This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.024176058914698324,
-    "train_runtime": 19938.5063,
+    "train_loss": 0.04123398642398541,
+    "train_runtime": 18693.2835,
     "train_samples": 1086,
-    "train_samples_per_second": 0.054,
+    "train_samples_per_second": 0.058,
     "train_steps_per_second": 0.002
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.024176058914698324,
-    "train_runtime": 19938.5063,
+    "train_loss": 0.04123398642398541,
+    "train_runtime": 18693.2835,
     "train_samples": 1086,
-    "train_samples_per_second": 0.054,
+    "train_samples_per_second": 0.058,
     "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -10,509 +10,509 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 3541.9607543945312,
+      "completion_length": 3301.4214477539062,
       "epoch": 0.02564102564102564,
-      "grad_norm": 5.551374912261963,
+      "grad_norm": 2.992680549621582,
       "kl": 0.0,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 0.3934539742767811,
-      "reward_std": 0.3425147756934166,
-      "rewards/code_reward": 0.3934539742767811,
+      "reward": 0.1030658227391541,
+      "reward_std": 0.20713394414633512,
+      "rewards/code_reward": 0.1030658227391541,
       "rewards/format_reward": 0.0,
       "step": 1
     },
     {
-      "completion_length": 3543.0106201171875,
+      "completion_length": 3303.0679321289062,
       "epoch": 0.05128205128205128,
-      "grad_norm": 1.4287736415863037,
+      "grad_norm": 1.2384196519851685,
       "kl": 0.0,
       "learning_rate": 5e-06,
       "loss": 0.0,
-      "reward": 0.46500000543892384,
-      "reward_std": 0.19661623612046242,
-      "rewards/code_reward": 0.46499999053776264,
+      "reward": 0.09688444854691625,
+      "reward_std": 0.16041716001927853,
+      "rewards/code_reward": 0.0968844504095614,
       "rewards/format_reward": 0.0,
       "step": 2
     },
     {
-      "completion_length": 3270.2929077148438,
+      "completion_length": 3189.9893188476562,
       "epoch": 0.07692307692307693,
-      "grad_norm": 2.6348750591278076,
-      "kl": 0.00787353515625,
+      "grad_norm": 10.27621841430664,
+      "kl": 0.004179954528808594,
       "learning_rate": 1e-05,
-      "loss": 0.0003,
-      "reward": 0.5235317498445511,
-      "reward_std": 0.30637360364198685,
-      "rewards/code_reward": 0.5235317498445511,
+      "loss": 0.0002,
+      "reward": 0.10514286020770669,
+      "reward_std": 0.2078455975279212,
+      "rewards/code_reward": 0.10514286020770669,
       "rewards/format_reward": 0.0,
       "step": 3
     },
     {
-      "completion_length": 2943.4500122070312,
+      "completion_length": 3000.7999267578125,
       "epoch": 0.10256410256410256,
-      "grad_norm": 0.7752403020858765,
-      "kl": 0.0511474609375,
+      "grad_norm": 2.5482373237609863,
+      "kl": 0.0092010498046875,
       "learning_rate": 9.98378869844137e-06,
-      "loss": 0.002,
-      "reward": 0.5904354751110077,
-      "reward_std": 0.21103981602936983,
-      "rewards/code_reward": 0.5904354676604271,
+      "loss": 0.0004,
+      "reward": 0.1636996939778328,
+      "reward_std": 0.28437332436442375,
+      "rewards/code_reward": 0.1636996977031231,
       "rewards/format_reward": 0.0,
       "step": 4
     },
     {
-      "completion_length": 3438.1785888671875,
+      "completion_length": 3725.6929321289062,
       "epoch": 0.1282051282051282,
-      "grad_norm": 3.3206803798675537,
-      "kl": 0.120361328125,
+      "grad_norm": 3.836555004119873,
+      "kl": 0.0565185546875,
       "learning_rate": 9.935271596564688e-06,
-      "loss": 0.0048,
-      "reward": 0.47174738347530365,
-      "reward_std": 0.1500780526548624,
-      "rewards/code_reward": 0.47174738347530365,
+      "loss": 0.0023,
+      "reward": 0.1552258525043726,
+      "reward_std": 0.20690688118338585,
+      "rewards/code_reward": 0.1552258525043726,
       "rewards/format_reward": 0.0,
       "step": 5
     },
     {
-      "completion_length": 3396.6856689453125,
+      "completion_length": 3449.982177734375,
       "epoch": 0.15384615384615385,
-      "grad_norm": 0.2983720600605011,
-      "kl": 0.2265625,
+      "grad_norm": 0.9245039224624634,
+      "kl": 0.153564453125,
       "learning_rate": 9.854798261200746e-06,
-      "loss": 0.0091,
-      "reward": 0.5083690956234932,
-      "reward_std": 0.23467476293444633,
-      "rewards/code_reward": 0.5083690956234932,
+      "loss": 0.0061,
+      "reward": 0.20291369408369064,
+      "reward_std": 0.2591838911175728,
+      "rewards/code_reward": 0.20291369780898094,
       "rewards/format_reward": 0.0,
       "step": 6
     },
     {
-      "completion_length": 3636.0071411132812,
+      "completion_length": 2533.5928955078125,
       "epoch": 0.1794871794871795,
-      "grad_norm": 0.29332682490348816,
-      "kl": 0.26611328125,
+      "grad_norm": 4.267428874969482,
+      "kl": 0.276611328125,
       "learning_rate": 9.74294850457488e-06,
-      "loss": 0.0106,
-      "reward": 0.387291356921196,
-      "reward_std": 0.23526490107178688,
-      "rewards/code_reward": 0.387291356921196,
+      "loss": 0.0111,
+      "reward": 0.09451580978929996,
+      "reward_std": 0.19575391709804535,
+      "rewards/code_reward": 0.09451580978929996,
       "rewards/format_reward": 0.0,
       "step": 7
     },
     {
-      "completion_length": 3375.800048828125,
+      "completion_length": 3610.7678833007812,
       "epoch": 0.20512820512820512,
-      "grad_norm": 0.3416775166988373,
-      "kl": 0.3359375,
+      "grad_norm": 0.9869590997695923,
+      "kl": 0.216552734375,
       "learning_rate": 9.600528206746613e-06,
-      "loss": 0.0134,
-      "reward": 0.48297154158353806,
-      "reward_std": 0.25456428155303,
-      "rewards/code_reward": 0.48297156393527985,
-      "rewards/format_reward": 0.0,
+      "loss": 0.0087,
+      "reward": 0.1328175999224186,
+      "reward_std": 0.24701963737607002,
+      "rewards/code_reward": 0.13246045634150505,
+      "rewards/format_reward": 0.0035714285913854837,
       "step": 8
     },
     {
-      "completion_length": 3155.9608154296875,
+      "completion_length": 3664.2072143554688,
       "epoch": 0.23076923076923078,
-      "grad_norm": 0.240010604262352,
-      "kl": 0.3779296875,
+      "grad_norm": 0.6975364685058594,
+      "kl": 0.23388671875,
       "learning_rate": 9.428563509225348e-06,
-      "loss": 0.0151,
-      "reward": 0.5900027677416801,
-      "reward_std": 0.18181878328323364,
-      "rewards/code_reward": 0.5900027677416801,
+      "loss": 0.0093,
+      "reward": 0.123524846509099,
+      "reward_std": 0.20174134522676468,
+      "rewards/code_reward": 0.123524846509099,
       "rewards/format_reward": 0.0,
       "step": 9
     },
     {
-      "completion_length": 3539.346435546875,
+      "completion_length": 3931.4750366210938,
       "epoch": 0.2564102564102564,
-      "grad_norm": 0.23563309013843536,
-      "kl": 0.3955078125,
+      "grad_norm": 0.7551573514938354,
+      "kl": 0.20751953125,
       "learning_rate": 9.22829342159729e-06,
-      "loss": 0.0158,
-      "reward": 0.4020918384194374,
-      "reward_std": 0.22648156061768532,
-      "rewards/code_reward": 0.4020918384194374,
+      "loss": 0.0083,
+      "reward": 0.08102564234286547,
+      "reward_std": 0.13897551037371159,
+      "rewards/code_reward": 0.08102564420551062,
       "rewards/format_reward": 0.0,
       "step": 10
     },
     {
-      "completion_length": 3615.935791015625,
+      "completion_length": 4086.3500366210938,
       "epoch": 0.28205128205128205,
-      "grad_norm": 0.19946347177028656,
-      "kl": 0.47216796875,
+      "grad_norm": 0.3929460942745209,
+      "kl": 0.201171875,
       "learning_rate": 9.001160894432979e-06,
-      "loss": 0.0189,
-      "reward": 0.41117217019200325,
-      "reward_std": 0.20069691445678473,
-      "rewards/code_reward": 0.41117217019200325,
+      "loss": 0.008,
+      "reward": 0.06611721590161324,
+      "reward_std": 0.1253935508430004,
+      "rewards/code_reward": 0.06611721590161324,
       "rewards/format_reward": 0.0,
       "step": 11
     },
     {
-      "completion_length": 3652.20361328125,
+      "completion_length": 4031.3786010742188,
       "epoch": 0.3076923076923077,
-      "grad_norm": 0.21409198641777039,
-      "kl": 0.556640625,
+      "grad_norm": 0.3564222455024719,
+      "kl": 0.208984375,
       "learning_rate": 8.748802422795361e-06,
-      "loss": 0.0222,
-      "reward": 0.41303257271647453,
-      "reward_std": 0.2046195026487112,
-      "rewards/code_reward": 0.41303258016705513,
+      "loss": 0.0084,
+      "reward": 0.09780338406562805,
+      "reward_std": 0.15440805070102215,
+      "rewards/code_reward": 0.09780338034033775,
       "rewards/format_reward": 0.0,
       "step": 12
     },
     {
-      "completion_length": 3682.6357421875,
+      "completion_length": 4077.70361328125,
       "epoch": 0.3333333333333333,
-      "grad_norm": 0.6496288180351257,
-      "kl": 0.580078125,
+      "grad_norm": 0.26154839992523193,
+      "kl": 0.215576171875,
       "learning_rate": 8.473036255255368e-06,
-      "loss": 0.0232,
-      "reward": 0.4388655610382557,
-      "reward_std": 0.22819811291992664,
-      "rewards/code_reward": 0.4388655610382557,
+      "loss": 0.0086,
+      "reward": 0.07981190085411072,
+      "reward_std": 0.1499769315123558,
+      "rewards/code_reward": 0.07981190085411072,
       "rewards/format_reward": 0.0,
       "step": 13
     },
     {
-      "completion_length": 3709.0142211914062,
+      "completion_length": 3955.335693359375,
       "epoch": 0.358974358974359,
-      "grad_norm": 0.23685197532176971,
-      "kl": 0.6474609375,
+      "grad_norm": 0.2279486507177353,
+      "kl": 0.22412109375,
       "learning_rate": 8.175849293369292e-06,
-      "loss": 0.0259,
-      "reward": 0.37063881754875183,
-      "reward_std": 0.23854901269078255,
-      "rewards/code_reward": 0.37063881754875183,
+      "loss": 0.009,
+      "reward": 0.08388392464257777,
+      "reward_std": 0.12758804112672806,
+      "rewards/code_reward": 0.08388392464257777,
       "rewards/format_reward": 0.0,
       "step": 14
     },
     {
-      "completion_length": 3758.8857421875,
+      "completion_length": 3789.925048828125,
       "epoch": 0.38461538461538464,
-      "grad_norm": 0.22902953624725342,
-      "kl": 0.66015625,
+      "grad_norm": 0.24723802506923676,
+      "kl": 0.227294921875,
       "learning_rate": 7.859382776007544e-06,
-      "loss": 0.0264,
-      "reward": 0.31185516342520714,
-      "reward_std": 0.22547182254493237,
-      "rewards/code_reward": 0.31185516342520714,
+      "loss": 0.0091,
+      "reward": 0.11268522241152823,
+      "reward_std": 0.18607539124786854,
+      "rewards/code_reward": 0.11268522613681853,
       "rewards/format_reward": 0.0,
       "step": 15
     },
     {
-      "completion_length": 3474.4249877929688,
+      "completion_length": 3539.6714477539062,
       "epoch": 0.41025641025641024,
-      "grad_norm": 0.2739468514919281,
-      "kl": 0.658203125,
+      "grad_norm": 0.2956843376159668,
+      "kl": 0.248779296875,
       "learning_rate": 7.52591685167953e-06,
-      "loss": 0.0263,
-      "reward": 0.46756455302238464,
-      "reward_std": 0.21291000582277775,
-      "rewards/code_reward": 0.46756456792354584,
+      "loss": 0.01,
+      "reward": 0.2600744627416134,
+      "reward_std": 0.21227671578526497,
+      "rewards/code_reward": 0.2600744664669037,
       "rewards/format_reward": 0.0,
       "step": 16
     },
     {
-      "completion_length": 3608.0107421875,
+      "completion_length": 3560.6357421875,
       "epoch": 0.4358974358974359,
-      "grad_norm": 0.26821818947792053,
-      "kl": 0.66796875,
+      "grad_norm": 0.3600841164588928,
+      "kl": 0.24560546875,
       "learning_rate": 7.1778541500113895e-06,
-      "loss": 0.0267,
-      "reward": 0.33532585576176643,
-      "reward_std": 0.29637256264686584,
-      "rewards/code_reward": 0.33532586693763733,
+      "loss": 0.0098,
+      "reward": 0.1853646468371153,
+      "reward_std": 0.23910802975296974,
+      "rewards/code_reward": 0.1853646282106638,
       "rewards/format_reward": 0.0,
       "step": 17
     },
     {
-      "completion_length": 3718.9964599609375,
+      "completion_length": 3478.9179077148438,
       "epoch": 0.46153846153846156,
-      "grad_norm": 0.24430078268051147,
-      "kl": 0.6259765625,
+      "grad_norm": 0.33355042338371277,
+      "kl": 0.243896484375,
       "learning_rate": 6.817702470744477e-06,
-      "loss": 0.025,
-      "reward": 0.22364513762295246,
-      "reward_std": 0.23529299348592758,
-      "rewards/code_reward": 0.22364513762295246,
+      "loss": 0.0098,
+      "reward": 0.1517396867275238,
+      "reward_std": 0.1984270103275776,
+      "rewards/code_reward": 0.15173968486487865,
       "rewards/format_reward": 0.0,
       "step": 18
     },
     {
-      "completion_length": 3312.9071044921875,
+      "completion_length": 3242.70361328125,
       "epoch": 0.48717948717948717,
-      "grad_norm": 0.2842939794063568,
-      "kl": 0.5859375,
+      "grad_norm": 0.4373861849308014,
+      "kl": 0.2568359375,
       "learning_rate": 6.448056714980768e-06,
-      "loss": 0.0234,
-      "reward": 0.46438145264983177,
-      "reward_std": 0.2700807861983776,
-      "rewards/code_reward": 0.46438145637512207,
+      "loss": 0.0103,
+      "reward": 0.25208037719130516,
+      "reward_std": 0.2785281799733639,
+      "rewards/code_reward": 0.25208036229014397,
       "rewards/format_reward": 0.0,
       "step": 19
     },
     {
-      "completion_length": 3342.6749877929688,
+      "completion_length": 3169.1214599609375,
       "epoch": 0.5128205128205128,
-      "grad_norm": 0.2545294761657715,
-      "kl": 0.5576171875,
+      "grad_norm": 0.44415220618247986,
+      "kl": 0.2646484375,
       "learning_rate": 6.071580188860955e-06,
-      "loss": 0.0223,
-      "reward": 0.5150031447410583,
-      "reward_std": 0.2922932505607605,
-      "rewards/code_reward": 0.5150031298398972,
+      "loss": 0.0106,
+      "reward": 0.2603498362004757,
+      "reward_std": 0.27874595671892166,
+      "rewards/code_reward": 0.2603498287498951,
       "rewards/format_reward": 0.0,
       "step": 20
     },
     {
-      "completion_length": 2976.3214111328125,
+      "completion_length": 2705.0072021484375,
       "epoch": 0.5384615384615384,
-      "grad_norm": 0.23293468356132507,
-      "kl": 0.6181640625,
+      "grad_norm": 0.4169292151927948,
+      "kl": 0.283203125,
       "learning_rate": 5.690985414382668e-06,
-      "loss": 0.0247,
-      "reward": 0.46903225034475327,
-      "reward_std": 0.23096787184476852,
-      "rewards/code_reward": 0.4690322279930115,
+      "loss": 0.0113,
+      "reward": 0.29475487023591995,
+      "reward_std": 0.29297249019145966,
+      "rewards/code_reward": 0.29475487396121025,
       "rewards/format_reward": 0.0,
       "step": 21
     },
     {
-      "completion_length": 3359.9892578125,
+      "completion_length": 2707.617919921875,
       "epoch": 0.5641025641025641,
-      "grad_norm": 0.2094847410917282,
-      "kl": 0.5703125,
+      "grad_norm": 3693.0712890625,
+      "kl": 29.77001953125,
       "learning_rate": 5.309014585617335e-06,
-      "loss": 0.0228,
-      "reward": 0.40152605809271336,
-      "reward_std": 0.2346283309161663,
-      "rewards/code_reward": 0.40152604319155216,
+      "loss": 1.1875,
+      "reward": 0.2492063995450735,
+      "reward_std": 0.25761475786566734,
+      "rewards/code_reward": 0.2492063995450735,
       "rewards/format_reward": 0.0,
       "step": 22
     },
     {
-      "completion_length": 2678.5321655273438,
+      "completion_length": 2344.096466064453,
       "epoch": 0.5897435897435898,
-      "grad_norm": 0.24114222824573517,
-      "kl": 0.5654296875,
+      "grad_norm": 0.38532546162605286,
+      "kl": 0.29150390625,
       "learning_rate": 4.928419811139046e-06,
-      "loss": 0.0227,
-      "reward": 0.608155146241188,
-      "reward_std": 0.14721081405878067,
-      "rewards/code_reward": 0.6081551611423492,
+      "loss": 0.0117,
+      "reward": 0.35850388184189796,
+      "reward_std": 0.2855670154094696,
+      "rewards/code_reward": 0.35850388184189796,
       "rewards/format_reward": 0.0,
       "step": 23
     },
     {
-      "completion_length": 2912.8821411132812,
+      "completion_length": 2637.9856567382812,
       "epoch": 0.6153846153846154,
-      "grad_norm": 0.3605176508426666,
-      "kl": 0.5908203125,
+      "grad_norm": 0.5114049315452576,
+      "kl": 0.326171875,
       "learning_rate": 4.551943285019233e-06,
-      "loss": 0.0236,
-      "reward": 0.5329480394721031,
-      "reward_std": 0.1990287434309721,
-      "rewards/code_reward": 0.5329480245709419,
+      "loss": 0.013,
+      "reward": 0.28699367493391037,
+      "reward_std": 0.2507447600364685,
+      "rewards/code_reward": 0.28699367120862007,
       "rewards/format_reward": 0.0,
       "step": 24
     },
     {
-      "completion_length": 3301.3857421875,
+      "completion_length": 2756.8214721679688,
       "epoch": 0.6410256410256411,
-      "grad_norm": 6.35435152053833,
-      "kl": 0.716796875,
+      "grad_norm": 0.483265221118927,
+      "kl": 0.357421875,
       "learning_rate": 4.182297529255525e-06,
-      "loss": 0.0287,
-      "reward": 0.36159154400229454,
-      "reward_std": 0.2206190638244152,
-      "rewards/code_reward": 0.36159154027700424,
+      "loss": 0.0143,
+      "reward": 0.22337647899985313,
+      "reward_std": 0.2525235004723072,
+      "rewards/code_reward": 0.22337647527456284,
       "rewards/format_reward": 0.0,
       "step": 25
     },
     {
-      "completion_length": 3339.2214965820312,
+      "completion_length": 2624.0642700195312,
       "epoch": 0.6666666666666666,
-      "grad_norm": 0.4265838861465454,
-      "kl": 0.7275390625,
+      "grad_norm": 0.44989556074142456,
+      "kl": 0.36669921875,
       "learning_rate": 3.822145849988612e-06,
-      "loss": 0.0291,
-      "reward": 0.3386395741254091,
-      "reward_std": 0.18014118261635303,
-      "rewards/code_reward": 0.3386395741254091,
+      "loss": 0.0147,
+      "reward": 0.2586754858493805,
+      "reward_std": 0.24251952394843102,
+      "rewards/code_reward": 0.2586754783987999,
       "rewards/format_reward": 0.0,
       "step": 26
     },
     {
-      "completion_length": 3056.1214599609375,
+      "completion_length": 2454.8429260253906,
       "epoch": 0.6923076923076923,
-      "grad_norm": 0.4635300636291504,
-      "kl": 0.7158203125,
+      "grad_norm": 0.3866746425628662,
+      "kl": 0.37109375,
       "learning_rate": 3.4740831483204696e-06,
-      "loss": 0.0286,
-      "reward": 0.4216299280524254,
-      "reward_std": 0.24011335149407387,
-      "rewards/code_reward": 0.4216299429535866,
+      "loss": 0.0148,
+      "reward": 0.2748478166759014,
+      "reward_std": 0.21565091237425804,
+      "rewards/code_reward": 0.2748478055000305,
       "rewards/format_reward": 0.0,
       "step": 27
     },
     {
-      "completion_length": 2741.1821899414062,
+      "completion_length": 2283.039306640625,
       "epoch": 0.717948717948718,
-      "grad_norm": 0.26547616720199585,
-      "kl": 0.6884765625,
+      "grad_norm": 0.34683477878570557,
+      "kl": 0.36376953125,
       "learning_rate": 3.1406172239924583e-06,
-      "loss": 0.0275,
-      "reward": 0.5913942456245422,
-      "reward_std": 0.19475560076534748,
-      "rewards/code_reward": 0.5913942456245422,
+      "loss": 0.0145,
+      "reward": 0.40325161814689636,
+      "reward_std": 0.29274996370077133,
+      "rewards/code_reward": 0.40325161069631577,
       "rewards/format_reward": 0.0,
       "step": 28
     },
     {
-      "completion_length": 3502.3857421875,
+      "completion_length": 2789.1107788085938,
       "epoch": 0.7435897435897436,
-      "grad_norm": 0.3135121464729309,
-      "kl": 0.8955078125,
+      "grad_norm": 0.6074942350387573,
+      "kl": 0.45703125,
       "learning_rate": 2.8241507066307106e-06,
-      "loss": 0.0358,
-      "reward": 0.33309811167418957,
-      "reward_std": 0.2021036557853222,
-      "rewards/code_reward": 0.3330980967730284,
+      "loss": 0.0183,
+      "reward": 0.23605629801750183,
+      "reward_std": 0.22556601464748383,
+      "rewards/code_reward": 0.23605630174279213,
       "rewards/format_reward": 0.0,
       "step": 29
     },
     {
-      "completion_length": 3053.2321166992188,
+      "completion_length": 2618.4750366210938,
       "epoch": 0.7692307692307693,
-      "grad_norm": 0.25711485743522644,
-      "kl": 0.880859375,
+      "grad_norm": 0.47094276547431946,
+      "kl": 0.4296875,
       "learning_rate": 2.526963744744635e-06,
-      "loss": 0.0352,
-      "reward": 0.44350775331258774,
-      "reward_std": 0.19116137735545635,
-      "rewards/code_reward": 0.44350775331258774,
+      "loss": 0.0172,
+      "reward": 0.24552644789218903,
+      "reward_std": 0.22016411647200584,
+      "rewards/code_reward": 0.24552644416689873,
       "rewards/format_reward": 0.0,
       "step": 30
     },
     {
-      "completion_length": 3173.0178833007812,
+      "completion_length": 2504.3535766601562,
       "epoch": 0.7948717948717948,
-      "grad_norm": 0.3827289044857025,
-      "kl": 0.8798828125,
+      "grad_norm": 0.460906058549881,
+      "kl": 0.390625,
       "learning_rate": 2.2511975772046403e-06,
-      "loss": 0.0352,
-      "reward": 0.42228348553180695,
-      "reward_std": 0.205778568983078,
-      "rewards/code_reward": 0.42228347808122635,
+      "loss": 0.0156,
+      "reward": 0.2590152435004711,
+      "reward_std": 0.24991632625460625,
+      "rewards/code_reward": 0.2590152509510517,
       "rewards/format_reward": 0.0,
       "step": 31
     },
     {
-      "completion_length": 3241.3535766601562,
+      "completion_length": 2634.0321655273438,
       "epoch": 0.8205128205128205,
-      "grad_norm": 0.4431661367416382,
-      "kl": 0.9033203125,
+      "grad_norm": 0.3826371729373932,
+      "kl": 0.4052734375,
       "learning_rate": 1.9988391055670234e-06,
-      "loss": 0.0362,
-      "reward": 0.3359471336007118,
-      "reward_std": 0.20435638166964054,
-      "rewards/code_reward": 0.3359471336007118,
+      "loss": 0.0162,
+      "reward": 0.2098755855113268,
+      "reward_std": 0.16958633810281754,
+      "rewards/code_reward": 0.2098755929619074,
       "rewards/format_reward": 0.0,
       "step": 32
     },
     {
-      "completion_length": 2643.057159423828,
+      "completion_length": 2563.767822265625,
       "epoch": 0.8461538461538461,
-      "grad_norm": 0.5116384029388428,
-      "kl": 0.8017578125,
+      "grad_norm": 0.40869322419166565,
+      "kl": 0.39990234375,
       "learning_rate": 1.771706578402711e-06,
-      "loss": 0.0321,
-      "reward": 0.5609605759382248,
-      "reward_std": 0.18299106322228909,
-      "rewards/code_reward": 0.5609605610370636,
+      "loss": 0.016,
+      "reward": 0.298298716545105,
+      "reward_std": 0.29596007615327835,
+      "rewards/code_reward": 0.29829873889684677,
       "rewards/format_reward": 0.0,
       "step": 33
     },
     {
-      "completion_length": 2872.3214721679688,
+      "completion_length": 2526.94287109375,
       "epoch": 0.8717948717948718,
-      "grad_norm": 0.44135168194770813,
-      "kl": 0.962890625,
+      "grad_norm": 0.45944005250930786,
+      "kl": 0.43359375,
       "learning_rate": 1.5714364907746535e-06,
-      "loss": 0.0385,
-      "reward": 0.41167889907956123,
-      "reward_std": 0.1887526996433735,
-      "rewards/code_reward": 0.41167885810136795,
+      "loss": 0.0173,
+      "reward": 0.2921798676252365,
+      "reward_std": 0.18791194818913937,
+      "rewards/code_reward": 0.2921798825263977,
       "rewards/format_reward": 0.0,
       "step": 34
     },
     {
-      "completion_length": 2978.9928588867188,
+      "completion_length": 2652.6107177734375,
       "epoch": 0.8974358974358975,
-      "grad_norm": 0.6080113053321838,
-      "kl": 1.0380859375,
+      "grad_norm": 0.42779994010925293,
+      "kl": 0.44091796875,
       "learning_rate": 1.399471793253389e-06,
-      "loss": 0.0416,
-      "reward": 0.3960940055549145,
-      "reward_std": 0.20829082280397415,
-      "rewards/code_reward": 0.3960940055549145,
+      "loss": 0.0176,
+      "reward": 0.20903684198856354,
+      "reward_std": 0.23266056552529335,
+      "rewards/code_reward": 0.20903684571385384,
       "rewards/format_reward": 0.0,
       "step": 35
     },
     {
-      "completion_length": 3258.403564453125,
+      "completion_length": 2777.0642700195312,
       "epoch": 0.9230769230769231,
-      "grad_norm": 0.41639065742492676,
-      "kl": 1.072265625,
+      "grad_norm": 0.48290908336639404,
+      "kl": 0.44189453125,
       "learning_rate": 1.257051495425121e-06,
-      "loss": 0.0429,
-      "reward": 0.1503874734044075,
-      "reward_std": 0.12068512104451656,
-      "rewards/code_reward": 0.15038747526705265,
+      "loss": 0.0177,
+      "reward": 0.07366262283176184,
+      "reward_std": 0.12753414548933506,
+      "rewards/code_reward": 0.07366262283176184,
       "rewards/format_reward": 0.0,
       "step": 36
     },
     {
-      "completion_length": 2731.4713745117188,
+      "completion_length": 2577.3999633789062,
       "epoch": 0.9487179487179487,
-      "grad_norm": 0.33002978563308716,
-      "kl": 1.0263671875,
+      "grad_norm": 0.41417568922042847,
+      "kl": 0.43896484375,
       "learning_rate": 1.1452017387992552e-06,
-      "loss": 0.041,
-      "reward": 0.234993327409029,
-      "reward_std": 0.1311760265380144,
-      "rewards/code_reward": 0.234993327409029,
+      "loss": 0.0176,
+      "reward": 0.12102217739447951,
+      "reward_std": 0.10151251405477524,
+      "rewards/code_reward": 0.12102217366918921,
       "rewards/format_reward": 0.0,
       "step": 37
     },
     {
-      "completion_length": 2790.7571411132812,
+      "completion_length": 2499.4678649902344,
       "epoch": 0.9743589743589743,
-      "grad_norm": 0.3518536388874054,
-      "kl": 1.005859375,
+      "grad_norm": 0.3276176452636719,
+      "kl": 0.4169921875,
       "learning_rate": 1.0647284034353122e-06,
-      "loss": 0.0402,
-      "reward": 0.28501298278570175,
-      "reward_std": 0.07613634853623807,
-      "rewards/code_reward": 0.28501297906041145,
+      "loss": 0.0167,
+      "reward": 0.16568297561025247,
+      "reward_std": 0.10975269501795992,
+      "rewards/code_reward": 0.16568297561025247,
       "rewards/format_reward": 0.0,
       "step": 38
     },
     {
-      "completion_length": 2785.1597290039062,
+      "completion_length": 2488.5069580078125,
       "epoch": 1.0,
-      "grad_norm": 0.6018698215484619,
-      "kl": 1.119140625,
+      "grad_norm": 0.42517927289009094,
+      "kl": 0.40771484375,
       "learning_rate": 1.0162113015586309e-06,
-      "loss": 0.0448,
-      "reward": 0.26603568717837334,
-      "reward_std": 0.09410964651033282,
-      "rewards/code_reward": 0.26603569462895393,
+      "loss": 0.0163,
+      "reward": 0.18103422969579697,
+      "reward_std": 0.11617903341539204,
+      "rewards/code_reward": 0.18103424459695816,
       "rewards/format_reward": 0.0,
       "step": 39
     },
@@ -520,9 +520,9 @@
       "epoch": 1.0,
       "step": 39,
       "total_flos": 0.0,
-      "train_loss": 0.024176058914698324,
-      "train_runtime": 19938.5063,
-      "train_samples_per_second": 0.054,
+      "train_loss": 0.04123398642398541,
+      "train_runtime": 18693.2835,
+      "train_samples_per_second": 0.058,
       "train_steps_per_second": 0.002
     }
   ],