Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: Qwen/Qwen2.5-Math-7B
-datasets: DigitalLearningGmbH/MATH-lighteval
 library_name: transformers
 model_name: Qwen-2.5-7B-Simple-RL
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen-2.5-7B-Simple-RL
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B) on the [DigitalLearningGmbH/MATH-lighteval](https://huggingface.co/datasets/DigitalLearningGmbH/MATH-lighteval) dataset.
+This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/hong-exploration/openr1/runs/2tj07458) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/hong-exploration/openr1/runs/mnu2rqhe) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -5,9 +5,9 @@
     "eval_samples_per_second": 1.117,
     "eval_steps_per_second": 0.01,
     "total_flos": 0.0,
-    "train_loss": -0.0005580486591372454,
-    "train_runtime": 7733.0464,
+    "train_loss": 0.0002975721653738533,
+    "train_runtime": 28855.0053,
     "train_samples": 7500,
-    "train_samples_per_second": 0.97,
-    "train_steps_per_second": 0.008
+    "train_samples_per_second": 0.52,
+    "train_steps_per_second": 0.004
 }

config.json

@@ -23,7 +23,7 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.49.0.dev0",
-  "use_cache": true,
+  "use_cache": false,
   "use_mrope": false,
   "use_sliding_window": false,
   "vocab_size": 152064

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c3a3b603b09af6380841e71c4bbeb6367b3b73a1eb8e48aea432e5ed7481080
+oid sha256:4fc7728d28ab847ff26d84462b68f9201534dbddb4212a0263658ee3cefa3e4b
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4fe5a8a76534589e305b7244c8016fb5e8e62c927910f9236c4464c20f35a0db
+oid sha256:64b2889819fdb5d36993f5a0c687d47c8c6c2ca9b8437ba4473a48fa6e15dfbc
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16fd3464bfdc897ecdd1b21bc00b473536a2a9220f09d2e9440755017ff81249
+oid sha256:84a3b102839dbbd28190c1d11454ea4380585e5b392932874f121b03667976fc
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62506b8566435edc32e43185c883eb83ca78188bfc138d8d299b046a821354f2
+oid sha256:97fa496342775957a94a7bd44fde2f410a4f3ad2c7e414ec1b949867850cf9be
 size 1089994880

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.0005580486591372454,
-    "train_runtime": 7733.0464,
+    "train_loss": 0.0002975721653738533,
+    "train_runtime": 28855.0053,
     "train_samples": 7500,
-    "train_samples_per_second": 0.97,
-    "train_steps_per_second": 0.008
+    "train_samples_per_second": 0.52,
+    "train_steps_per_second": 0.004
 }

trainer_state.json

@@ -1,176 +1,374 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9893390191897654,
-  "eval_steps": 100,
-  "global_step": 58,
+  "epoch": 1.9893390191897655,
+  "eval_steps": 39,
+  "global_step": 116,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 621.0189994812011,
+      "epoch": 0,
+      "eval_completion_length": 580.933360529403,
+      "eval_kl": 0.0,
+      "eval_loss": 1.623958087293431e-05,
+      "eval_reward": 0.5275844605966878,
+      "eval_reward_std": 0.35651993932434545,
+      "eval_rewards/accuracy_reward": 0.5264149032080897,
+      "eval_rewards/format_reward": 0.0011695573350396782,
+      "eval_runtime": 4528.6778,
+      "eval_samples_per_second": 1.104,
+      "eval_steps_per_second": 0.01,
+      "step": 0
+    },
+    {
+      "completion_length": 604.259627532959,
       "epoch": 0.08528784648187633,
-      "grad_norm": 1.747524380683899,
-      "kl": 0.00022215843200683593,
-      "learning_rate": 2.5e-06,
+      "grad_norm": 1.9525558948516846,
+      "kl": 0.00012731552124023438,
+      "learning_rate": 1.25e-06,
       "loss": 0.0,
-      "reward": 0.614508954435587,
-      "reward_std": 0.34517091177403925,
-      "rewards/accuracy_reward": 0.6138393096625805,
-      "rewards/format_reward": 0.0006696428870782256,
+      "reward": 0.6582589574158192,
+      "reward_std": 0.3278909016400576,
+      "rewards/accuracy_reward": 0.6573660992085933,
+      "rewards/format_reward": 0.0008928571827709675,
       "step": 5
     },
     {
-      "completion_length": 628.56743850708,
+      "completion_length": 616.6596267700195,
       "epoch": 0.17057569296375266,
-      "grad_norm": 0.463044136762619,
-      "kl": 0.003224754333496094,
-      "learning_rate": 2.956412726139078e-06,
+      "grad_norm": 1.6776472330093384,
+      "kl": 0.0016300678253173828,
+      "learning_rate": 2.5e-06,
       "loss": 0.0001,
-      "reward": 0.6892857477068901,
-      "reward_std": 0.28228127732872965,
-      "rewards/accuracy_reward": 0.6892857477068901,
-      "rewards/format_reward": 0.0,
+      "reward": 0.6910714641213417,
+      "reward_std": 0.3150623720139265,
+      "rewards/accuracy_reward": 0.6908482506871223,
+      "rewards/format_reward": 0.00022321429569274187,
       "step": 10
     },
     {
-      "completion_length": 629.7674423217774,
+      "completion_length": 640.9160995483398,
       "epoch": 0.255863539445629,
-      "grad_norm": 0.13988655805587769,
-      "kl": 0.0027944564819335936,
-      "learning_rate": 2.7836719084521715e-06,
-      "loss": 0.0001,
-      "reward": 0.7198661088943481,
-      "reward_std": 0.2151865204796195,
-      "rewards/accuracy_reward": 0.7198661088943481,
+      "grad_norm": 0.13862861692905426,
+      "kl": 0.004425430297851562,
+      "learning_rate": 2.9938448364256362e-06,
+      "loss": 0.0002,
+      "reward": 0.7236607506871223,
+      "reward_std": 0.26740851644426583,
+      "rewards/accuracy_reward": 0.7236607506871223,
       "rewards/format_reward": 0.0,
       "step": 15
     },
     {
-      "completion_length": 588.3330619812011,
+      "completion_length": 631.7879745483399,
       "epoch": 0.3411513859275053,
-      "grad_norm": 0.11190886050462723,
-      "kl": 0.0031805038452148438,
-      "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0001,
-      "reward": 0.7776786029338837,
-      "reward_std": 0.19100845558568835,
-      "rewards/accuracy_reward": 0.7774553894996643,
-      "rewards/format_reward": 0.00022321429569274187,
+      "grad_norm": 0.13688388466835022,
+      "kl": 0.007097053527832031,
+      "learning_rate": 2.956412726139078e-06,
+      "loss": 0.0003,
+      "reward": 0.7348214671015739,
+      "reward_std": 0.22995020002126693,
+      "rewards/accuracy_reward": 0.7348214671015739,
+      "rewards/format_reward": 0.0,
       "step": 20
     },
     {
-      "completion_length": 604.8185546875,
+      "completion_length": 619.988638305664,
       "epoch": 0.42643923240938164,
-      "grad_norm": 0.23962590098381042,
-      "kl": 0.003417205810546875,
-      "learning_rate": 2.1156192081791355e-06,
-      "loss": 0.0001,
-      "reward": 0.7651786044239998,
-      "reward_std": 0.190818705689162,
-      "rewards/accuracy_reward": 0.7651786044239998,
+      "grad_norm": 0.8423397541046143,
+      "kl": 0.004146957397460937,
+      "learning_rate": 2.88581929876693e-06,
+      "loss": 0.0002,
+      "reward": 0.7430803924798965,
+      "reward_std": 0.22932490669190883,
+      "rewards/accuracy_reward": 0.7430803924798965,
       "rewards/format_reward": 0.0,
       "step": 25
     },
     {
-      "completion_length": 608.9339538574219,
+      "completion_length": 606.4966804504395,
       "epoch": 0.511727078891258,
-      "grad_norm": 0.18276575207710266,
-      "kl": 0.0049468994140625,
-      "learning_rate": 1.6808050203829845e-06,
+      "grad_norm": 0.11532779783010483,
+      "kl": 0.00457305908203125,
+      "learning_rate": 2.7836719084521715e-06,
       "loss": 0.0002,
-      "reward": 0.7654018238186836,
-      "reward_std": 0.18531882837414743,
-      "rewards/accuracy_reward": 0.7654018238186836,
+      "reward": 0.752455385029316,
+      "reward_std": 0.206120328605175,
+      "rewards/accuracy_reward": 0.752455385029316,
       "rewards/format_reward": 0.0,
       "step": 30
     },
     {
-      "completion_length": 602.1547149658203,
+      "completion_length": 589.5859703063965,
       "epoch": 0.5970149253731343,
-      "grad_norm": 0.20332862436771393,
-      "kl": 0.005048370361328125,
-      "learning_rate": 1.2296174432791415e-06,
+      "grad_norm": 0.10156127065420151,
+      "kl": 0.004618453979492188,
+      "learning_rate": 2.652296367060421e-06,
       "loss": 0.0002,
-      "reward": 0.7700893253087997,
-      "reward_std": 0.19738825485110284,
-      "rewards/accuracy_reward": 0.7700893253087997,
+      "reward": 0.783928607404232,
+      "reward_std": 0.19030314590781927,
+      "rewards/accuracy_reward": 0.783928607404232,
       "rewards/format_reward": 0.0,
       "step": 35
     },
     {
-      "completion_length": 595.8953392028809,
+      "epoch": 0.6652452025586354,
+      "eval_completion_length": 582.3961823836617,
+      "eval_kl": 0.005152472896852355,
+      "eval_loss": 0.00021490754443220794,
+      "eval_reward": 0.6949793272260306,
+      "eval_reward_std": 0.20541082589306695,
+      "eval_rewards/accuracy_reward": 0.6949793272260306,
+      "eval_rewards/format_reward": 0.0,
+      "eval_runtime": 4509.8983,
+      "eval_samples_per_second": 1.109,
+      "eval_steps_per_second": 0.01,
+      "step": 39
+    },
+    {
+      "completion_length": 591.9185485839844,
       "epoch": 0.6823027718550106,
-      "grad_norm": 0.16811978816986084,
-      "kl": 0.003607177734375,
-      "learning_rate": 8.029152419343472e-07,
-      "loss": 0.0001,
-      "reward": 0.7645089626312256,
-      "reward_std": 0.17381814373657106,
-      "rewards/accuracy_reward": 0.7645089626312256,
+      "grad_norm": 0.12734617292881012,
+      "kl": 0.005767822265625,
+      "learning_rate": 2.4946839873611927e-06,
+      "loss": 0.0002,
+      "reward": 0.7678571790456772,
+      "reward_std": 0.2127251597121358,
+      "rewards/accuracy_reward": 0.7678571790456772,
       "rewards/format_reward": 0.0,
       "step": 40
     },
     {
-      "completion_length": 589.4821655273438,
+      "completion_length": 592.6504753112793,
       "epoch": 0.767590618336887,
-      "grad_norm": 0.10356087237596512,
-      "kl": 0.003725433349609375,
-      "learning_rate": 4.3933982822017883e-07,
-      "loss": 0.0001,
-      "reward": 0.7756696790456772,
-      "reward_std": 0.18397480640560387,
-      "rewards/accuracy_reward": 0.7756696790456772,
+      "grad_norm": 0.08878223598003387,
+      "kl": 0.00388336181640625,
+      "learning_rate": 2.314423473302218e-06,
+      "loss": 0.0002,
+      "reward": 0.7566964656114579,
+      "reward_std": 0.1911870390176773,
+      "rewards/accuracy_reward": 0.7566964656114579,
       "rewards/format_reward": 0.0,
       "step": 45
     },
     {
-      "completion_length": 593.6323921203614,
+      "completion_length": 579.0308303833008,
       "epoch": 0.8528784648187633,
-      "grad_norm": 0.21458302438259125,
-      "kl": 0.003731536865234375,
-      "learning_rate": 1.718159615201853e-07,
-      "loss": 0.0001,
-      "reward": 0.7703125327825546,
-      "reward_std": 0.17003230415284634,
-      "rewards/accuracy_reward": 0.7703125327825546,
+      "grad_norm": 0.10042494535446167,
+      "kl": 0.006903076171875,
+      "learning_rate": 2.1156192081791355e-06,
+      "loss": 0.0003,
+      "reward": 0.7850446835160255,
+      "reward_std": 0.17857401072978973,
+      "rewards/accuracy_reward": 0.7850446835160255,
       "rewards/format_reward": 0.0,
       "step": 50
     },
     {
-      "completion_length": 600.3533706665039,
+      "completion_length": 574.0143119812012,
       "epoch": 0.9381663113006397,
-      "grad_norm": 0.1505906730890274,
-      "kl": 0.003759002685546875,
-      "learning_rate": 2.4570139579284723e-08,
-      "loss": 0.0002,
-      "reward": 0.781026816368103,
-      "reward_std": 0.20167170464992523,
-      "rewards/accuracy_reward": 0.781026816368103,
+      "grad_norm": 0.10724026709794998,
+      "kl": 0.00340118408203125,
+      "learning_rate": 1.9027978012115653e-06,
+      "loss": 0.0001,
+      "reward": 0.7665178909897804,
+      "reward_std": 0.16513762380927802,
+      "rewards/accuracy_reward": 0.7665178909897804,
       "rewards/format_reward": 0.0,
       "step": 55
     },
     {
-      "completion_length": 575.5026321411133,
-      "epoch": 0.9893390191897654,
-      "kl": 0.0039323170979817705,
-      "reward": 0.7875744427243868,
-      "reward_std": 0.15916253595302501,
-      "rewards/accuracy_reward": 0.7875744427243868,
+      "completion_length": 575.9379244717685,
+      "epoch": 1.0341151385927505,
+      "grad_norm": 0.07216066867113113,
+      "kl": 0.0033409812233664775,
+      "learning_rate": 1.6808050203829845e-06,
+      "loss": 0.0001,
+      "reward": 0.7869318506934426,
+      "reward_std": 0.18544116904112426,
+      "rewards/accuracy_reward": 0.7869318506934426,
+      "rewards/format_reward": 0.0,
+      "step": 60
+    },
+    {
+      "completion_length": 565.3899772644043,
+      "epoch": 1.1194029850746268,
+      "grad_norm": 0.09482983499765396,
+      "kl": 0.0034759521484375,
+      "learning_rate": 1.454695458298667e-06,
+      "loss": 0.0001,
+      "reward": 0.7991071745753289,
+      "reward_std": 0.17083449736237527,
+      "rewards/accuracy_reward": 0.7991071745753289,
+      "rewards/format_reward": 0.0,
+      "step": 65
+    },
+    {
+      "completion_length": 571.7268089294433,
+      "epoch": 1.2046908315565032,
+      "grad_norm": 0.08009294420480728,
+      "kl": 0.00347137451171875,
+      "learning_rate": 1.2296174432791415e-06,
+      "loss": 0.0001,
+      "reward": 0.7839286103844643,
+      "reward_std": 0.17667411090806126,
+      "rewards/accuracy_reward": 0.7837053954601287,
+      "rewards/format_reward": 0.00022321429569274187,
+      "step": 70
+    },
+    {
+      "completion_length": 565.4739059448242,
+      "epoch": 1.2899786780383795,
+      "grad_norm": 0.06680043786764145,
+      "kl": 0.00329742431640625,
+      "learning_rate": 1.0106958161686963e-06,
+      "loss": 0.0001,
+      "reward": 0.7879464671015739,
+      "reward_std": 0.1701756412163377,
+      "rewards/accuracy_reward": 0.7879464671015739,
+      "rewards/format_reward": 0.0,
+      "step": 75
+    },
+    {
+      "epoch": 1.3411513859275053,
+      "eval_completion_length": 560.9655020060101,
+      "eval_kl": 0.004444744891158197,
+      "eval_loss": 0.00021425970771815628,
+      "eval_reward": 0.6994489503012921,
+      "eval_reward_std": 0.208146112796903,
+      "eval_rewards/accuracy_reward": 0.6994489503012921,
+      "eval_rewards/format_reward": 0.0,
+      "eval_runtime": 4435.898,
+      "eval_samples_per_second": 1.127,
+      "eval_steps_per_second": 0.01,
+      "step": 78
+    },
+    {
+      "completion_length": 551.9654197692871,
+      "epoch": 1.375266524520256,
+      "grad_norm": 0.1283424198627472,
+      "kl": 0.0043430328369140625,
+      "learning_rate": 8.029152419343472e-07,
+      "loss": 0.0002,
+      "reward": 0.7924107536673546,
+      "reward_std": 0.17128037009388208,
+      "rewards/accuracy_reward": 0.7924107536673546,
+      "rewards/format_reward": 0.0,
+      "step": 80
+    },
+    {
+      "completion_length": 541.5716796875,
+      "epoch": 1.4605543710021323,
+      "grad_norm": 0.3723917603492737,
+      "kl": 0.003989791870117188,
+      "learning_rate": 6.11006712953975e-07,
+      "loss": 0.0002,
+      "reward": 0.8220982536673546,
+      "reward_std": 0.15211040936410428,
+      "rewards/accuracy_reward": 0.8220982536673546,
+      "rewards/format_reward": 0.0,
+      "step": 85
+    },
+    {
+      "completion_length": 563.2312767028809,
+      "epoch": 1.5458422174840085,
+      "grad_norm": 0.09019841253757477,
+      "kl": 0.0033016204833984375,
+      "learning_rate": 4.3933982822017883e-07,
+      "loss": 0.0001,
+      "reward": 0.7973214641213417,
+      "reward_std": 0.17157677421346307,
+      "rewards/accuracy_reward": 0.7973214641213417,
+      "rewards/format_reward": 0.0,
+      "step": 90
+    },
+    {
+      "completion_length": 554.155606842041,
+      "epoch": 1.6311300639658848,
+      "grad_norm": 0.06605476140975952,
+      "kl": 0.0035243988037109374,
+      "learning_rate": 2.9182330117358096e-07,
+      "loss": 0.0001,
+      "reward": 0.799107177555561,
+      "reward_std": 0.15023288973607124,
+      "rewards/accuracy_reward": 0.799107177555561,
+      "rewards/format_reward": 0.0,
+      "step": 95
+    },
+    {
+      "completion_length": 562.7194396972657,
+      "epoch": 1.716417910447761,
+      "grad_norm": 0.09438558667898178,
+      "kl": 0.003635406494140625,
+      "learning_rate": 1.718159615201853e-07,
+      "loss": 0.0001,
+      "reward": 0.7908482491970062,
+      "reward_std": 0.1763416165485978,
+      "rewards/accuracy_reward": 0.7908482491970062,
+      "rewards/format_reward": 0.0,
+      "step": 100
+    },
+    {
+      "completion_length": 581.4640892028808,
+      "epoch": 1.8017057569296375,
+      "grad_norm": 0.08443931490182877,
+      "kl": 0.003319549560546875,
+      "learning_rate": 8.20502774480395e-08,
+      "loss": 0.0001,
+      "reward": 0.7691964648663998,
+      "reward_std": 0.178659126162529,
+      "rewards/accuracy_reward": 0.7691964648663998,
+      "rewards/format_reward": 0.0,
+      "step": 105
+    },
+    {
+      "completion_length": 578.9288177490234,
+      "epoch": 1.886993603411514,
+      "grad_norm": 0.09037923067808151,
+      "kl": 0.0033283233642578125,
+      "learning_rate": 2.4570139579284723e-08,
+      "loss": 0.0001,
+      "reward": 0.7825893223285675,
+      "reward_std": 0.16389768896624446,
+      "rewards/accuracy_reward": 0.7825893223285675,
+      "rewards/format_reward": 0.0,
+      "step": 110
+    },
+    {
+      "completion_length": 573.8774856567383,
+      "epoch": 1.9722814498933903,
+      "grad_norm": 0.06668173521757126,
+      "kl": 0.0031665802001953126,
+      "learning_rate": 6.843232656998933e-10,
+      "loss": 0.0001,
+      "reward": 0.7763393208384514,
+      "reward_std": 0.15611171433702112,
+      "rewards/accuracy_reward": 0.7763393208384514,
+      "rewards/format_reward": 0.0,
+      "step": 115
+    },
+    {
+      "completion_length": 572.2143058776855,
+      "epoch": 1.9893390191897655,
+      "kl": 0.00342559814453125,
+      "reward": 0.7700893357396126,
+      "reward_std": 0.18100517243146896,
+      "rewards/accuracy_reward": 0.7700893357396126,
       "rewards/format_reward": 0.0,
-      "step": 58,
+      "step": 116,
       "total_flos": 0.0,
-      "train_loss": -0.0005580486591372454,
-      "train_runtime": 7733.0464,
-      "train_samples_per_second": 0.97,
-      "train_steps_per_second": 0.008
+      "train_loss": 0.0002975721653738533,
+      "train_runtime": 28855.0053,
+      "train_samples_per_second": 0.52,
+      "train_steps_per_second": 0.004
     }
   ],
   "logging_steps": 5,
-  "max_steps": 58,
+  "max_steps": 116,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 2,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6fe83cf439fa999af0a2fd93d7b931e23c9657bdc43a674526b6cee7c5f10839
+oid sha256:6537740e58151835a485586f2a6eaa1a299d34fc0dc65230787068aa7f3ab238
 size 7544