Model save

README.md

@@ -0,0 +1,68 @@
+---
+base_model: allenai/OLMoE-1B-7B-0125-Instruct
+library_name: transformers
+model_name: OLMoE-1B-7B-0125-Instruct-grpo
+tags:
+- generated_from_trainer
+- trl
+- grpo
+licence: license
+---
+
+# Model Card for OLMoE-1B-7B-0125-Instruct-grpo
+
+This model is a fine-tuned version of [allenai/OLMoE-1B-7B-0125-Instruct](https://huggingface.co/allenai/OLMoE-1B-7B-0125-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="chenggong1995/OLMoE-1B-7B-0125-Instruct-grpo", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/gongc1995-city-university-of-hong-kong/huggingface/runs/papxj9zr) 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.16.0.dev0
+- Transformers: 4.49.0.dev0
+- Pytorch: 2.5.1
+- Datasets: 3.2.0
+- Tokenizers: 0.21.0
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{zhihong2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.01271514138510042,
+    "train_runtime": 147461.7668,
+    "train_samples": 72441,
+    "train_samples_per_second": 0.491,
+    "train_steps_per_second": 0.004
+}

config.json

@@ -0,0 +1,33 @@
+{
+  "_name_or_path": "allenai/OLMoE-1B-7B-0125-Instruct",
+  "architectures": [
+    "OlmoeForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "clip_qkv": null,
+  "eos_token_id": 50279,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 4096,
+  "model_type": "olmoe",
+  "norm_topk_prob": false,
+  "num_attention_heads": 16,
+  "num_experts": 64,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "output_router_logits": false,
+  "pad_token_id": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.01,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.49.0.dev0",
+  "use_cache": false,
+  "vocab_size": 50304
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 50279,
+  "pad_token_id": 1,
+  "transformers_version": "4.49.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1909a6771bf5f61c6d117bd479d14bc6a180db3ffe32bd2c09746e2d445972e
+size 4997744872

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44c276968b28dc5a27fe503f3bc4fdfdfaedc77b7011657a0c1d212b54082e89
+size 4997235176

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:998814a0196413ffbdd822f2d4beb069f44ef1cf932a3aefd3bf488fa15c3baf
+size 3843741912

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "|||IP_ADDRESS|||",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "|||IP_ADDRESS|||",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,249 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50280": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "|||IP_ADDRESS|||",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>\n' + message['content'] + '\n' }}{% elif message['role'] == 'user' %}{{ '<|user|>\n' + message['content'] + '\n' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>\n'  + message['content'] + eos_token + '\n' }}{% else %}{{ '<|assistant|>\n'  + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "|||IP_ADDRESS|||",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.01271514138510042,
+    "train_runtime": 147461.7668,
+    "train_samples": 72441,
+    "train_samples_per_second": 0.491,
+    "train_steps_per_second": 0.004
+}

trainer_state.json

@@ -0,0 +1,1809 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9987438399845395,
+  "eval_steps": 100,
+  "global_step": 646,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 457.96500358581545,
+      "epoch": 0.007730215479756498,
+      "grad_norm": 0.7608509038009253,
+      "kl": 0.0007220268249511719,
+      "learning_rate": 7.692307692307694e-07,
+      "loss": 0.0,
+      "reward": 0.45107144135981797,
+      "reward_std": 0.4365754766389728,
+      "rewards/accuracy_reward": 0.04785714372992515,
+      "rewards/format_reward": 0.4032142972573638,
+      "step": 5
+    },
+    {
+      "completion_length": 445.5060745239258,
+      "epoch": 0.015460430959512996,
+      "grad_norm": 1.5113540423097132,
+      "kl": 0.006823968887329101,
+      "learning_rate": 1.5384615384615387e-06,
+      "loss": 0.0003,
+      "reward": 0.5625000160187483,
+      "reward_std": 0.4424866631627083,
+      "rewards/accuracy_reward": 0.06107142954133451,
+      "rewards/format_reward": 0.5014285854995251,
+      "step": 10
+    },
+    {
+      "completion_length": 411.183215713501,
+      "epoch": 0.023190646439269495,
+      "grad_norm": 1.5933497538626709,
+      "kl": 0.029457855224609374,
+      "learning_rate": 2.307692307692308e-06,
+      "loss": 0.0012,
+      "reward": 0.7785714413970709,
+      "reward_std": 0.35438590599223974,
+      "rewards/accuracy_reward": 0.05464285819325596,
+      "rewards/format_reward": 0.7239285875111818,
+      "step": 15
+    },
+    {
+      "completion_length": 381.2946424484253,
+      "epoch": 0.03092086191902599,
+      "grad_norm": 0.4776993449956818,
+      "kl": 0.02753753662109375,
+      "learning_rate": 3.0769230769230774e-06,
+      "loss": 0.0011,
+      "reward": 0.8642857238650322,
+      "reward_std": 0.29294114708900454,
+      "rewards/accuracy_reward": 0.03821428623050451,
+      "rewards/format_reward": 0.8260714389383793,
+      "step": 20
+    },
+    {
+      "completion_length": 362.5117889404297,
+      "epoch": 0.03865107739878249,
+      "grad_norm": 0.635286407672086,
+      "kl": 0.03197479248046875,
+      "learning_rate": 3.846153846153847e-06,
+      "loss": 0.0013,
+      "reward": 0.9035714380443096,
+      "reward_std": 0.23572507686913013,
+      "rewards/accuracy_reward": 0.032142857718281445,
+      "rewards/format_reward": 0.8714285790920258,
+      "step": 25
+    },
+    {
+      "completion_length": 315.9125024795532,
+      "epoch": 0.04638129287853899,
+      "grad_norm": 0.3737341489393726,
+      "kl": 0.042462158203125,
+      "learning_rate": 4.615384615384616e-06,
+      "loss": 0.0017,
+      "reward": 0.9617857217788697,
+      "reward_std": 0.1909911316819489,
+      "rewards/accuracy_reward": 0.04142857207916677,
+      "rewards/format_reward": 0.920357147604227,
+      "step": 30
+    },
+    {
+      "completion_length": 303.57821617126467,
+      "epoch": 0.054111508358295486,
+      "grad_norm": 0.29770380771379046,
+      "kl": 0.05437469482421875,
+      "learning_rate": 5.384615384615385e-06,
+      "loss": 0.0022,
+      "reward": 0.975714298337698,
+      "reward_std": 0.16020548259839415,
+      "rewards/accuracy_reward": 0.03321428613271564,
+      "rewards/format_reward": 0.942500003427267,
+      "step": 35
+    },
+    {
+      "completion_length": 327.77964363098147,
+      "epoch": 0.06184172383805198,
+      "grad_norm": 0.4560285277274068,
+      "kl": 0.056903076171875,
+      "learning_rate": 6.153846153846155e-06,
+      "loss": 0.0023,
+      "reward": 1.016785728931427,
+      "reward_std": 0.1489170640707016,
+      "rewards/accuracy_reward": 0.05392857214901596,
+      "rewards/format_reward": 0.9628571450710297,
+      "step": 40
+    },
+    {
+      "completion_length": 365.1317882537842,
+      "epoch": 0.06957193931780849,
+      "grad_norm": 0.31719927124196806,
+      "kl": 0.0534454345703125,
+      "learning_rate": 6.923076923076923e-06,
+      "loss": 0.0021,
+      "reward": 1.0375000171363353,
+      "reward_std": 0.19086767928674817,
+      "rewards/accuracy_reward": 0.08285714453086257,
+      "rewards/format_reward": 0.9546428598463536,
+      "step": 45
+    },
+    {
+      "completion_length": 358.1814300537109,
+      "epoch": 0.07730215479756498,
+      "grad_norm": 0.38771993742220967,
+      "kl": 0.06278076171875,
+      "learning_rate": 7.692307692307694e-06,
+      "loss": 0.0025,
+      "reward": 1.040357157588005,
+      "reward_std": 0.21823968663811683,
+      "rewards/accuracy_reward": 0.0939285738626495,
+      "rewards/format_reward": 0.9464285746216774,
+      "step": 50
+    },
+    {
+      "completion_length": 338.32928562164307,
+      "epoch": 0.08503237027732148,
+      "grad_norm": 0.3009616852428193,
+      "kl": 0.07071533203125,
+      "learning_rate": 8.461538461538462e-06,
+      "loss": 0.0028,
+      "reward": 1.0582143023610115,
+      "reward_std": 0.1873143390752375,
+      "rewards/accuracy_reward": 0.09785714475437998,
+      "rewards/format_reward": 0.9603571452200412,
+      "step": 55
+    },
+    {
+      "completion_length": 355.8407180786133,
+      "epoch": 0.09276258575707798,
+      "grad_norm": 0.325964246595786,
+      "kl": 0.07974853515625,
+      "learning_rate": 9.230769230769232e-06,
+      "loss": 0.0032,
+      "reward": 1.0664285972714425,
+      "reward_std": 0.1719504727050662,
+      "rewards/accuracy_reward": 0.09464285941794515,
+      "rewards/format_reward": 0.9717857159674168,
+      "step": 60
+    },
+    {
+      "completion_length": 385.9732172012329,
+      "epoch": 0.10049280123683448,
+      "grad_norm": 0.29390759666087973,
+      "kl": 0.095562744140625,
+      "learning_rate": 1e-05,
+      "loss": 0.0038,
+      "reward": 1.1042857334017753,
+      "reward_std": 0.22559863626956939,
+      "rewards/accuracy_reward": 0.14392857477068902,
+      "rewards/format_reward": 0.9603571452200412,
+      "step": 65
+    },
+    {
+      "completion_length": 361.5596435546875,
+      "epoch": 0.10822301671659097,
+      "grad_norm": 0.3746410425327759,
+      "kl": 0.115264892578125,
+      "learning_rate": 9.99817273706943e-06,
+      "loss": 0.0046,
+      "reward": 1.1235714480280876,
+      "reward_std": 0.20448200711980463,
+      "rewards/accuracy_reward": 0.1439285755623132,
+      "rewards/format_reward": 0.9796428583562374,
+      "step": 70
+    },
+    {
+      "completion_length": 348.6992870330811,
+      "epoch": 0.11595323219634747,
+      "grad_norm": 0.3978179778651091,
+      "kl": 0.148199462890625,
+      "learning_rate": 9.992692283833639e-06,
+      "loss": 0.0059,
+      "reward": 1.154285728931427,
+      "reward_std": 0.25590373622253537,
+      "rewards/accuracy_reward": 0.19571429127827286,
+      "rewards/format_reward": 0.9585714310407638,
+      "step": 75
+    },
+    {
+      "completion_length": 360.27392997741697,
+      "epoch": 0.12368344767610397,
+      "grad_norm": 0.3285728265102151,
+      "kl": 0.122698974609375,
+      "learning_rate": 9.983562645984248e-06,
+      "loss": 0.0049,
+      "reward": 1.108214306086302,
+      "reward_std": 0.207218668051064,
+      "rewards/accuracy_reward": 0.1371428606333211,
+      "rewards/format_reward": 0.9710714302957058,
+      "step": 80
+    },
+    {
+      "completion_length": 326.5525007247925,
+      "epoch": 0.13141366315586048,
+      "grad_norm": 0.3701658784343864,
+      "kl": 0.13370361328125,
+      "learning_rate": 9.970790496420781e-06,
+      "loss": 0.0053,
+      "reward": 1.1421428725123406,
+      "reward_std": 0.22562065320089458,
+      "rewards/accuracy_reward": 0.17250000457279385,
+      "rewards/format_reward": 0.9696428589522839,
+      "step": 85
+    },
+    {
+      "completion_length": 340.90464515686034,
+      "epoch": 0.13914387863561697,
+      "grad_norm": 0.3053892797015973,
+      "kl": 0.197650146484375,
+      "learning_rate": 9.954385170373415e-06,
+      "loss": 0.0079,
+      "reward": 1.2007143080234528,
+      "reward_std": 0.223483042512089,
+      "rewards/accuracy_reward": 0.22071429125498981,
+      "rewards/format_reward": 0.9800000011920929,
+      "step": 90
+    },
+    {
+      "completion_length": 370.76178760528563,
+      "epoch": 0.14687409411537347,
+      "grad_norm": 0.26661678402016015,
+      "kl": 0.15225830078125,
+      "learning_rate": 9.934358658579808e-06,
+      "loss": 0.0061,
+      "reward": 1.2128571577370166,
+      "reward_std": 0.24409643467515707,
+      "rewards/accuracy_reward": 0.24714286350645126,
+      "rewards/format_reward": 0.9657142877578735,
+      "step": 95
+    },
+    {
+      "completion_length": 383.44642810821534,
+      "epoch": 0.15460430959512997,
+      "grad_norm": 0.24333486200642604,
+      "kl": 0.140423583984375,
+      "learning_rate": 9.910725598521014e-06,
+      "loss": 0.0056,
+      "reward": 1.1667857356369495,
+      "reward_std": 0.22021743915975095,
+      "rewards/accuracy_reward": 0.19857143445406109,
+      "rewards/format_reward": 0.968214287608862,
+      "step": 100
+    },
+    {
+      "epoch": 0.15460430959512997,
+      "eval_completion_length": 356.63817596435547,
+      "eval_kl": 0.14697265625,
+      "eval_loss": 0.005899494513869286,
+      "eval_reward": 1.2053571343421936,
+      "eval_reward_std": 0.25127829425036907,
+      "eval_rewards/accuracy_reward": 0.23571429401636124,
+      "eval_rewards/format_reward": 0.9696428775787354,
+      "eval_runtime": 85.1811,
+      "eval_samples_per_second": 1.162,
+      "eval_steps_per_second": 0.023,
+      "step": 100
+    },
+    {
+      "completion_length": 381.14428634643554,
+      "epoch": 0.16233452507488647,
+      "grad_norm": 0.32426868486126564,
+      "kl": 0.153460693359375,
+      "learning_rate": 9.883503263722864e-06,
+      "loss": 0.0061,
+      "reward": 1.2335714444518089,
+      "reward_std": 0.23903345353901387,
+      "rewards/accuracy_reward": 0.25892857937142255,
+      "rewards/format_reward": 0.9746428586542606,
+      "step": 105
+    },
+    {
+      "completion_length": 369.8639308929443,
+      "epoch": 0.17006474055464296,
+      "grad_norm": 0.2662792930753043,
+      "kl": 0.16746826171875,
+      "learning_rate": 9.852711551130662e-06,
+      "loss": 0.0067,
+      "reward": 1.2200000181794166,
+      "reward_std": 0.21895755929872393,
+      "rewards/accuracy_reward": 0.24428572261240333,
+      "rewards/format_reward": 0.9757142871618271,
+      "step": 110
+    },
+    {
+      "completion_length": 336.4314300537109,
+      "epoch": 0.17779495603439946,
+      "grad_norm": 0.3050936417250258,
+      "kl": 0.1807861328125,
+      "learning_rate": 9.818372966566404e-06,
+      "loss": 0.0072,
+      "reward": 1.201428586244583,
+      "reward_std": 0.2133496488444507,
+      "rewards/accuracy_reward": 0.22321429231669754,
+      "rewards/format_reward": 0.9782142870128154,
+      "step": 115
+    },
+    {
+      "completion_length": 350.5678596496582,
+      "epoch": 0.18552517151415596,
+      "grad_norm": 0.34458458483928667,
+      "kl": 0.19677734375,
+      "learning_rate": 9.780512608279156e-06,
+      "loss": 0.0079,
+      "reward": 1.146071445941925,
+      "reward_std": 0.2668243307620287,
+      "rewards/accuracy_reward": 0.2042857193853706,
+      "rewards/format_reward": 0.9417857177555561,
+      "step": 120
+    },
+    {
+      "completion_length": 433.7532165527344,
+      "epoch": 0.19325538699391245,
+      "grad_norm": 0.36013965861594144,
+      "kl": 0.220751953125,
+      "learning_rate": 9.739158148600609e-06,
+      "loss": 0.0088,
+      "reward": 1.0971428729593753,
+      "reward_std": 0.3263824795372784,
+      "rewards/accuracy_reward": 0.21000000613275915,
+      "rewards/format_reward": 0.887142863869667,
+      "step": 125
+    },
+    {
+      "completion_length": 497.63500156402586,
+      "epoch": 0.20098560247366895,
+      "grad_norm": 0.34690086218654914,
+      "kl": 0.248583984375,
+      "learning_rate": 9.694339813719243e-06,
+      "loss": 0.0099,
+      "reward": 1.1189285896718502,
+      "reward_std": 0.2914966750890017,
+      "rewards/accuracy_reward": 0.19928571924101562,
+      "rewards/format_reward": 0.9196428619325161,
+      "step": 130
+    },
+    {
+      "completion_length": 326.64821701049806,
+      "epoch": 0.20871581795342545,
+      "grad_norm": 0.33711598936424164,
+      "kl": 0.22991943359375,
+      "learning_rate": 9.646090361587828e-06,
+      "loss": 0.0092,
+      "reward": 1.2050000175833702,
+      "reward_std": 0.23595014372840523,
+      "rewards/accuracy_reward": 0.23321429167408497,
+      "rewards/format_reward": 0.9717857159674168,
+      "step": 135
+    },
+    {
+      "completion_length": 337.5925010681152,
+      "epoch": 0.21644603343318194,
+      "grad_norm": 19.330410175568485,
+      "kl": 0.3224609375,
+      "learning_rate": 9.594445057980483e-06,
+      "loss": 0.0129,
+      "reward": 1.2114285856485367,
+      "reward_std": 0.2494897256605327,
+      "rewards/accuracy_reward": 0.2425000074552372,
+      "rewards/format_reward": 0.9689285732805729,
+      "step": 140
+    },
+    {
+      "completion_length": 359.734645652771,
+      "epoch": 0.22417624891293844,
+      "grad_norm": 1.318993891026559,
+      "kl": 0.32476806640625,
+      "learning_rate": 9.539441650716744e-06,
+      "loss": 0.013,
+      "reward": 1.1532142981886864,
+      "reward_std": 0.268144457321614,
+      "rewards/accuracy_reward": 0.22357143508270383,
+      "rewards/format_reward": 0.9296428613364697,
+      "step": 145
+    },
+    {
+      "completion_length": 308.5442869186401,
+      "epoch": 0.23190646439269494,
+      "grad_norm": 5.642497910156749,
+      "kl": 0.4279296875,
+      "learning_rate": 9.481120342071462e-06,
+      "loss": 0.0171,
+      "reward": 1.1817857310175897,
+      "reward_std": 0.2727494416758418,
+      "rewards/accuracy_reward": 0.23892857865430414,
+      "rewards/format_reward": 0.9428571462631226,
+      "step": 150
+    },
+    {
+      "completion_length": 271.07821559906006,
+      "epoch": 0.23963667987245144,
+      "grad_norm": 12.266848353099874,
+      "kl": 0.36064453125,
+      "learning_rate": 9.41952375939078e-06,
+      "loss": 0.0144,
+      "reward": 1.2067857295274735,
+      "reward_std": 0.2593365713953972,
+      "rewards/accuracy_reward": 0.24535714988596738,
+      "rewards/format_reward": 0.9614285737276077,
+      "step": 155
+    },
+    {
+      "completion_length": 275.08464469909666,
+      "epoch": 0.24736689535220793,
+      "grad_norm": 0.34495859229810133,
+      "kl": 0.226123046875,
+      "learning_rate": 9.354696923935573e-06,
+      "loss": 0.009,
+      "reward": 1.2053571574389934,
+      "reward_std": 0.25177041683346035,
+      "rewards/accuracy_reward": 0.24000000536907465,
+      "rewards/format_reward": 0.9653571449220181,
+      "step": 160
+    },
+    {
+      "completion_length": 250.24928703308106,
+      "epoch": 0.25509711083196446,
+      "grad_norm": 0.3975356570351778,
+      "kl": 0.30623779296875,
+      "learning_rate": 9.28668721797517e-06,
+      "loss": 0.0123,
+      "reward": 1.2103571623563767,
+      "reward_std": 0.2850337457843125,
+      "rewards/accuracy_reward": 0.26000000804197043,
+      "rewards/format_reward": 0.9503571458160878,
+      "step": 165
+    },
+    {
+      "completion_length": 240.8703582763672,
+      "epoch": 0.26282732631172095,
+      "grad_norm": 15.560140664497903,
+      "kl": 0.58157958984375,
+      "learning_rate": 9.215544350155423e-06,
+      "loss": 0.0233,
+      "reward": 1.1975000113248826,
+      "reward_std": 0.23778518568724394,
+      "rewards/accuracy_reward": 0.23892857872415335,
+      "rewards/format_reward": 0.9585714310407638,
+      "step": 170
+    },
+    {
+      "completion_length": 244.13857307434083,
+      "epoch": 0.27055754179147745,
+      "grad_norm": 0.32982217891891835,
+      "kl": 0.33167724609375,
+      "learning_rate": 9.141320319166387e-06,
+      "loss": 0.0133,
+      "reward": 1.2264285899698735,
+      "reward_std": 0.19883861131966113,
+      "rewards/accuracy_reward": 0.24464286372531205,
+      "rewards/format_reward": 0.9817857153713703,
+      "step": 175
+    },
+    {
+      "completion_length": 250.61321525573732,
+      "epoch": 0.27828775727123395,
+      "grad_norm": 0.6994861767616407,
+      "kl": 0.28465576171875,
+      "learning_rate": 9.064069375736216e-06,
+      "loss": 0.0114,
+      "reward": 1.1882142946124077,
+      "reward_std": 0.24957463573664426,
+      "rewards/accuracy_reward": 0.23000000652391464,
+      "rewards/format_reward": 0.9582142882049084,
+      "step": 180
+    },
+    {
+      "completion_length": 172.02142906188965,
+      "epoch": 0.28601797275099045,
+      "grad_norm": 1.042105326460384,
+      "kl": 0.7489013671875,
+      "learning_rate": 8.983847982979022e-06,
+      "loss": 0.03,
+      "reward": 1.0496428698301314,
+      "reward_std": 0.3774342490360141,
+      "rewards/accuracy_reward": 0.18750000472646206,
+      "rewards/format_reward": 0.8621428653597831,
+      "step": 185
+    },
+    {
+      "completion_length": 202.3928581237793,
+      "epoch": 0.29374818823074694,
+      "grad_norm": 0.5558652862204951,
+      "kl": 0.478271484375,
+      "learning_rate": 8.900714775125696e-06,
+      "loss": 0.0191,
+      "reward": 1.119642873853445,
+      "reward_std": 0.2985226408578455,
+      "rewards/accuracy_reward": 0.20392857666593045,
+      "rewards/format_reward": 0.9157142907381057,
+      "step": 190
+    },
+    {
+      "completion_length": 254.10214443206786,
+      "epoch": 0.30147840371050344,
+      "grad_norm": 0.37080340908454873,
+      "kl": 0.543505859375,
+      "learning_rate": 8.814730514667841e-06,
+      "loss": 0.0217,
+      "reward": 1.193214301019907,
+      "reward_std": 0.20587898194789886,
+      "rewards/accuracy_reward": 0.22071429206989707,
+      "rewards/format_reward": 0.9725000016391278,
+      "step": 195
+    },
+    {
+      "completion_length": 239.24893054962158,
+      "epoch": 0.30920861919025994,
+      "grad_norm": 1.419820098954415,
+      "kl": 0.29251708984375,
+      "learning_rate": 8.725958047946155e-06,
+      "loss": 0.0117,
+      "reward": 1.2103571616113187,
+      "reward_std": 0.2034143740311265,
+      "rewards/accuracy_reward": 0.21821429126430303,
+      "rewards/format_reward": 0.9921428576111794,
+      "step": 200
+    },
+    {
+      "epoch": 0.30920861919025994,
+      "eval_completion_length": 248.33312606811523,
+      "eval_kl": 0.275390625,
+      "eval_loss": 0.011179111897945404,
+      "eval_reward": 1.2339285761117935,
+      "eval_reward_std": 0.21089003048837185,
+      "eval_rewards/accuracy_reward": 0.24285714887082577,
+      "eval_rewards/format_reward": 0.9910714328289032,
+      "eval_runtime": 65.1713,
+      "eval_samples_per_second": 1.519,
+      "eval_steps_per_second": 0.031,
+      "step": 200
+    },
+    {
+      "completion_length": 277.3242883682251,
+      "epoch": 0.31693883467001643,
+      "grad_norm": 0.4847949666125137,
+      "kl": 0.28834228515625,
+      "learning_rate": 8.634462259215719e-06,
+      "loss": 0.0115,
+      "reward": 1.224642877280712,
+      "reward_std": 0.21538263475522398,
+      "rewards/accuracy_reward": 0.2310714353341609,
+      "rewards/format_reward": 0.9935714289546013,
+      "step": 205
+    },
+    {
+      "completion_length": 312.77535858154295,
+      "epoch": 0.32466905014977293,
+      "grad_norm": 0.2890010677054343,
+      "kl": 0.29903564453125,
+      "learning_rate": 8.540310023221751e-06,
+      "loss": 0.012,
+      "reward": 1.2089285902678966,
+      "reward_std": 0.21994048971682786,
+      "rewards/accuracy_reward": 0.22392857696395368,
+      "rewards/format_reward": 0.9850000008940697,
+      "step": 210
+    },
+    {
+      "completion_length": 326.0450016021729,
+      "epoch": 0.33239926562952943,
+      "grad_norm": 1.5857581308410953,
+      "kl": 0.3697021484375,
+      "learning_rate": 8.443570156320518e-06,
+      "loss": 0.0148,
+      "reward": 1.2025000154972076,
+      "reward_std": 0.23657457055523992,
+      "rewards/accuracy_reward": 0.22392857796512544,
+      "rewards/format_reward": 0.978571429848671,
+      "step": 215
+    },
+    {
+      "completion_length": 316.9417865753174,
+      "epoch": 0.3401294811092859,
+      "grad_norm": 0.3354478326906057,
+      "kl": 0.335546875,
+      "learning_rate": 8.344313366181096e-06,
+      "loss": 0.0134,
+      "reward": 1.2114285811781884,
+      "reward_std": 0.22043001046404243,
+      "rewards/accuracy_reward": 0.22142857748549433,
+      "rewards/format_reward": 0.9900000005960464,
+      "step": 220
+    },
+    {
+      "completion_length": 322.1989326477051,
+      "epoch": 0.3478596965890424,
+      "grad_norm": 0.3533516561360887,
+      "kl": 0.2961669921875,
+      "learning_rate": 8.242612200104779e-06,
+      "loss": 0.0118,
+      "reward": 1.1957143053412438,
+      "reward_std": 0.1864888435229659,
+      "rewards/accuracy_reward": 0.20000000530853868,
+      "rewards/format_reward": 0.9957142859697342,
+      "step": 225
+    },
+    {
+      "completion_length": 350.877858543396,
+      "epoch": 0.3555899120687989,
+      "grad_norm": 0.2820220582755007,
+      "kl": 0.29697265625,
+      "learning_rate": 8.138540991999871e-06,
+      "loss": 0.0119,
+      "reward": 1.1785714492201804,
+      "reward_std": 0.2050869233906269,
+      "rewards/accuracy_reward": 0.18892857611645014,
+      "rewards/format_reward": 0.989642857760191,
+      "step": 230
+    },
+    {
+      "completion_length": 401.6375032424927,
+      "epoch": 0.3633201275485554,
+      "grad_norm": 3.0020500774885437,
+      "kl": 0.28485107421875,
+      "learning_rate": 8.032175808050659e-06,
+      "loss": 0.0114,
+      "reward": 1.1100000217556953,
+      "reward_std": 0.1967215022072196,
+      "rewards/accuracy_reward": 0.13000000335741788,
+      "rewards/format_reward": 0.9800000011920929,
+      "step": 235
+    },
+    {
+      "completion_length": 355.40107440948486,
+      "epoch": 0.3710503430283119,
+      "grad_norm": 0.28586570118341365,
+      "kl": 0.25643310546875,
+      "learning_rate": 7.923594391120237e-06,
+      "loss": 0.0103,
+      "reward": 1.182142873108387,
+      "reward_std": 0.25666751638054847,
+      "rewards/accuracy_reward": 0.2135714341653511,
+      "rewards/format_reward": 0.9685714304447174,
+      "step": 240
+    },
+    {
+      "completion_length": 343.7985729217529,
+      "epoch": 0.3787805585080684,
+      "grad_norm": 0.31296482492061484,
+      "kl": 0.2564697265625,
+      "learning_rate": 7.812876103927845e-06,
+      "loss": 0.0103,
+      "reward": 1.2296428754925728,
+      "reward_std": 0.24015271971002222,
+      "rewards/accuracy_reward": 0.26000000685453417,
+      "rewards/format_reward": 0.9696428589522839,
+      "step": 245
+    },
+    {
+      "completion_length": 370.3635730743408,
+      "epoch": 0.3865107739878249,
+      "grad_norm": 0.29554233915571987,
+      "kl": 0.308056640625,
+      "learning_rate": 7.700101871042254e-06,
+      "loss": 0.0123,
+      "reward": 1.2103571586310864,
+      "reward_std": 0.24133061729371547,
+      "rewards/accuracy_reward": 0.24000000657979398,
+      "rewards/format_reward": 0.9703571446239948,
+      "step": 250
+    },
+    {
+      "completion_length": 360.6953601837158,
+      "epoch": 0.3942409894675814,
+      "grad_norm": 0.2413456510082092,
+      "kl": 0.24384765625,
+      "learning_rate": 7.5853541197335716e-06,
+      "loss": 0.0098,
+      "reward": 1.2132143005728722,
+      "reward_std": 0.22292996691539885,
+      "rewards/accuracy_reward": 0.23035714915022254,
+      "rewards/format_reward": 0.9828571438789367,
+      "step": 255
+    },
+    {
+      "completion_length": 350.62500114440917,
+      "epoch": 0.4019712049473379,
+      "grad_norm": 0.37205800913968085,
+      "kl": 0.2304443359375,
+      "learning_rate": 7.468716719726733e-06,
+      "loss": 0.0092,
+      "reward": 1.2350000202655793,
+      "reward_std": 0.22238717656582593,
+      "rewards/accuracy_reward": 0.25107143486384303,
+      "rewards/format_reward": 0.9839285723865032,
+      "step": 260
+    },
+    {
+      "completion_length": 345.0939315795898,
+      "epoch": 0.4097014204270944,
+      "grad_norm": 0.5062737177381504,
+      "kl": 0.476416015625,
+      "learning_rate": 7.3502749219006795e-06,
+      "loss": 0.0191,
+      "reward": 1.2007143013179302,
+      "reward_std": 0.24597819074988364,
+      "rewards/accuracy_reward": 0.23928572090808303,
+      "rewards/format_reward": 0.9614285737276077,
+      "step": 265
+    },
+    {
+      "completion_length": 352.0803602218628,
+      "epoch": 0.4174316359068509,
+      "grad_norm": 0.6322459279226639,
+      "kl": 0.49012451171875,
+      "learning_rate": 7.230115295978048e-06,
+      "loss": 0.0196,
+      "reward": 1.1835714481770991,
+      "reward_std": 0.25889406390488146,
+      "rewards/accuracy_reward": 0.23107143563684077,
+      "rewards/format_reward": 0.9525000028312206,
+      "step": 270
+    },
+    {
+      "completion_length": 329.8617877960205,
+      "epoch": 0.4251618513866074,
+      "grad_norm": 0.6668664351542097,
+      "kl": 0.54197998046875,
+      "learning_rate": 7.10832566725092e-06,
+      "loss": 0.0217,
+      "reward": 1.2160714395344256,
+      "reward_std": 0.2636732104234397,
+      "rewards/accuracy_reward": 0.25285714818164706,
+      "rewards/format_reward": 0.9632142879068851,
+      "step": 275
+    },
+    {
+      "completion_length": 332.9325021743774,
+      "epoch": 0.4328920668663639,
+      "grad_norm": 0.862765307967942,
+      "kl": 0.3385986328125,
+      "learning_rate": 6.984995052388856e-06,
+      "loss": 0.0136,
+      "reward": 1.216785727441311,
+      "reward_std": 0.22093856362625958,
+      "rewards/accuracy_reward": 0.2428571493830532,
+      "rewards/format_reward": 0.9739285729825496,
+      "step": 280
+    },
+    {
+      "completion_length": 312.248931312561,
+      "epoch": 0.4406222823461204,
+      "grad_norm": 0.24545660293906987,
+      "kl": 0.2172607421875,
+      "learning_rate": 6.860213594376153e-06,
+      "loss": 0.0087,
+      "reward": 1.2532143011689185,
+      "reward_std": 0.2206911792047322,
+      "rewards/accuracy_reward": 0.2650000074878335,
+      "rewards/format_reward": 0.988214286416769,
+      "step": 285
+    },
+    {
+      "completion_length": 332.121431350708,
+      "epoch": 0.4483524978258769,
+      "grad_norm": 0.26547805533093827,
+      "kl": 0.21544189453125,
+      "learning_rate": 6.734072496625867e-06,
+      "loss": 0.0086,
+      "reward": 1.2425000101327897,
+      "reward_std": 0.22103301845490933,
+      "rewards/accuracy_reward": 0.2614285790361464,
+      "rewards/format_reward": 0.9810714296996593,
+      "step": 290
+    },
+    {
+      "completion_length": 375.60142974853517,
+      "epoch": 0.4560827133056334,
+      "grad_norm": 0.3864015818375869,
+      "kl": 0.29462890625,
+      "learning_rate": 6.606663956318782e-06,
+      "loss": 0.0118,
+      "reward": 1.2335714407265186,
+      "reward_std": 0.2652750977315009,
+      "rewards/accuracy_reward": 0.27428572135977447,
+      "rewards/format_reward": 0.9592857167124749,
+      "step": 295
+    },
+    {
+      "completion_length": 374.28750228881836,
+      "epoch": 0.4638129287853899,
+      "grad_norm": 0.3911583369947814,
+      "kl": 0.4628662109375,
+      "learning_rate": 6.478081097015985e-06,
+      "loss": 0.0185,
+      "reward": 1.2142857298254968,
+      "reward_std": 0.2580754934810102,
+      "rewards/accuracy_reward": 0.26392857884056864,
+      "rewards/format_reward": 0.9503571458160878,
+      "step": 300
+    },
+    {
+      "epoch": 0.4638129287853899,
+      "eval_completion_length": 332.5970458984375,
+      "eval_kl": 0.37158203125,
+      "eval_loss": 0.015323814935982227,
+      "eval_reward": 1.2125000059604645,
+      "eval_reward_std": 0.2901066914200783,
+      "eval_rewards/accuracy_reward": 0.25000000558793545,
+      "eval_rewards/format_reward": 0.9625000134110451,
+      "eval_runtime": 79.9,
+      "eval_samples_per_second": 1.239,
+      "eval_steps_per_second": 0.025,
+      "step": 300
+    },
+    {
+      "completion_length": 343.74143199920655,
+      "epoch": 0.4715431442651464,
+      "grad_norm": 0.38709550861776243,
+      "kl": 0.419091796875,
+      "learning_rate": 6.348417900594408e-06,
+      "loss": 0.0168,
+      "reward": 1.2289285875856877,
+      "reward_std": 0.24545602239668368,
+      "rewards/accuracy_reward": 0.2603571491781622,
+      "rewards/format_reward": 0.9685714304447174,
+      "step": 305
+    },
+    {
+      "completion_length": 334.96392917633057,
+      "epoch": 0.47927335974490287,
+      "grad_norm": 0.4392077094147235,
+      "kl": 0.39736328125,
+      "learning_rate": 6.2177691385549595e-06,
+      "loss": 0.0159,
+      "reward": 1.2221428737044335,
+      "reward_std": 0.21934431809931992,
+      "rewards/accuracy_reward": 0.24250000559259205,
+      "rewards/format_reward": 0.9796428583562374,
+      "step": 310
+    },
+    {
+      "completion_length": 308.76142902374266,
+      "epoch": 0.48700357522465937,
+      "grad_norm": 0.32886450863598027,
+      "kl": 0.333636474609375,
+      "learning_rate": 6.086230302753563e-06,
+      "loss": 0.0133,
+      "reward": 1.2160714440047742,
+      "reward_std": 0.19036251278594135,
+      "rewards/accuracy_reward": 0.23107143498491495,
+      "rewards/format_reward": 0.9850000008940697,
+      "step": 315
+    },
+    {
+      "completion_length": 333.30928802490234,
+      "epoch": 0.49473379070441587,
+      "grad_norm": 1.3946586911147487,
+      "kl": 0.37911376953125,
+      "learning_rate": 5.95389753560565e-06,
+      "loss": 0.0152,
+      "reward": 1.218928585201502,
+      "reward_std": 0.2294909401796758,
+      "rewards/accuracy_reward": 0.24178572059608996,
+      "rewards/format_reward": 0.9771428585052491,
+      "step": 320
+    },
+    {
+      "completion_length": 340.629288482666,
+      "epoch": 0.5024640061841724,
+      "grad_norm": 0.46150339120676787,
+      "kl": 0.36011962890625,
+      "learning_rate": 5.820867559815186e-06,
+      "loss": 0.0144,
+      "reward": 1.236071441322565,
+      "reward_std": 0.21350146597251296,
+      "rewards/accuracy_reward": 0.26500000632368026,
+      "rewards/format_reward": 0.9710714302957058,
+      "step": 325
+    },
+    {
+      "completion_length": 330.6417865753174,
+      "epoch": 0.5101942216639289,
+      "grad_norm": 0.7086286073905499,
+      "kl": 0.37000732421875,
+      "learning_rate": 5.68723760767954e-06,
+      "loss": 0.0148,
+      "reward": 1.2314285844564439,
+      "reward_std": 0.24753115382045507,
+      "rewards/accuracy_reward": 0.263571435585618,
+      "rewards/format_reward": 0.9678571447730064,
+      "step": 330
+    },
+    {
+      "completion_length": 306.0692878723145,
+      "epoch": 0.5179244371436854,
+      "grad_norm": 0.4592159227361041,
+      "kl": 0.43155517578125,
+      "learning_rate": 5.55310535002189e-06,
+      "loss": 0.0173,
+      "reward": 1.2125000201165677,
+      "reward_std": 0.2561903980560601,
+      "rewards/accuracy_reward": 0.24892857796512544,
+      "rewards/format_reward": 0.9635714307427407,
+      "step": 335
+    },
+    {
+      "completion_length": 301.78607177734375,
+      "epoch": 0.5256546526234419,
+      "grad_norm": 1.3334554805750072,
+      "kl": 0.40533447265625,
+      "learning_rate": 5.418568824803121e-06,
+      "loss": 0.0162,
+      "reward": 1.2046428814530372,
+      "reward_std": 0.24458126006647946,
+      "rewards/accuracy_reward": 0.23857143591158092,
+      "rewards/format_reward": 0.966071430593729,
+      "step": 340
+    },
+    {
+      "completion_length": 311.9282154083252,
+      "epoch": 0.5333848681031984,
+      "grad_norm": 0.49010519701579414,
+      "kl": 0.47899169921875,
+      "learning_rate": 5.2837263654653715e-06,
+      "loss": 0.0192,
+      "reward": 1.2089285902678966,
+      "reward_std": 0.24385176142677664,
+      "rewards/accuracy_reward": 0.24214286329224705,
+      "rewards/format_reward": 0.96678571626544,
+      "step": 345
+    },
+    {
+      "completion_length": 356.58785762786863,
+      "epoch": 0.5411150835829549,
+      "grad_norm": 0.40401739909507167,
+      "kl": 0.297998046875,
+      "learning_rate": 5.148676529059604e-06,
+      "loss": 0.0119,
+      "reward": 1.2339285902678967,
+      "reward_std": 0.2254973877221346,
+      "rewards/accuracy_reward": 0.25178572128061205,
+      "rewards/format_reward": 0.9821428582072258,
+      "step": 350
+    },
+    {
+      "completion_length": 401.15607299804685,
+      "epoch": 0.5488452990627114,
+      "grad_norm": 0.43919579917421436,
+      "kl": 0.467236328125,
+      "learning_rate": 5.013518024209757e-06,
+      "loss": 0.0187,
+      "reward": 1.223928590863943,
+      "reward_std": 0.25751951709389687,
+      "rewards/accuracy_reward": 0.2525000067660585,
+      "rewards/format_reward": 0.9714285731315613,
+      "step": 355
+    },
+    {
+      "completion_length": 372.611074256897,
+      "epoch": 0.5565755145424679,
+      "grad_norm": 0.5922777261314072,
+      "kl": 0.43697509765625,
+      "learning_rate": 4.878349638966098e-06,
+      "loss": 0.0175,
+      "reward": 1.2207143008708954,
+      "reward_std": 0.2579284094274044,
+      "rewards/accuracy_reward": 0.25178572118747977,
+      "rewards/format_reward": 0.9689285732805729,
+      "step": 360
+    },
+    {
+      "completion_length": 351.6828565597534,
+      "epoch": 0.5643057300222244,
+      "grad_norm": 0.29650599886416334,
+      "kl": 0.3662109375,
+      "learning_rate": 4.743270168600523e-06,
+      "loss": 0.0146,
+      "reward": 1.2132143080234528,
+      "reward_std": 0.21272512217983602,
+      "rewards/accuracy_reward": 0.23607143440749495,
+      "rewards/format_reward": 0.9771428585052491,
+      "step": 365
+    },
+    {
+      "completion_length": 352.63428859710694,
+      "epoch": 0.5720359455019809,
+      "grad_norm": 0.2713235178095771,
+      "kl": 0.32996826171875,
+      "learning_rate": 4.608378343396581e-06,
+      "loss": 0.0132,
+      "reward": 1.2214285895228385,
+      "reward_std": 0.2412548642605543,
+      "rewards/accuracy_reward": 0.24821429257281125,
+      "rewards/format_reward": 0.9732142873108387,
+      "step": 370
+    },
+    {
+      "completion_length": 400.2235725402832,
+      "epoch": 0.5797661609817374,
+      "grad_norm": 0.6530373844158506,
+      "kl": 0.534814453125,
+      "learning_rate": 4.47377275648701e-06,
+      "loss": 0.0214,
+      "reward": 1.1946428716182709,
+      "reward_std": 0.2843459703028202,
+      "rewards/accuracy_reward": 0.24785715015605092,
+      "rewards/format_reward": 0.9467857174575329,
+      "step": 375
+    },
+    {
+      "completion_length": 384.1235717773437,
+      "epoch": 0.5874963764614939,
+      "grad_norm": 0.5620739725200156,
+      "kl": 0.433917236328125,
+      "learning_rate": 4.33955179179149e-06,
+      "loss": 0.0174,
+      "reward": 1.1839285880327224,
+      "reward_std": 0.27014439897611736,
+      "rewards/accuracy_reward": 0.2300000066170469,
+      "rewards/format_reward": 0.9539285741746426,
+      "step": 380
+    },
+    {
+      "completion_length": 367.36893062591554,
+      "epoch": 0.5952265919412504,
+      "grad_norm": 0.3417788309196838,
+      "kl": 0.45970458984375,
+      "learning_rate": 4.205813552107337e-06,
+      "loss": 0.0184,
+      "reward": 1.2242857284843922,
+      "reward_std": 0.2591902747750282,
+      "rewards/accuracy_reward": 0.26214286310132595,
+      "rewards/format_reward": 0.9621428593993187,
+      "step": 385
+    },
+    {
+      "completion_length": 344.9664316177368,
+      "epoch": 0.6029568074210069,
+      "grad_norm": 0.383725508536428,
+      "kl": 0.2947509765625,
+      "learning_rate": 4.0726557874056625e-06,
+      "loss": 0.0118,
+      "reward": 1.1982143066823483,
+      "reward_std": 0.21319182245060803,
+      "rewards/accuracy_reward": 0.21928572067990898,
+      "rewards/format_reward": 0.9789285726845265,
+      "step": 390
+    },
+    {
+      "completion_length": 325.8582160949707,
+      "epoch": 0.6106870229007634,
+      "grad_norm": 0.5862839492831149,
+      "kl": 0.36025390625,
+      "learning_rate": 3.940175823385412e-06,
+      "loss": 0.0144,
+      "reward": 1.2221428655087947,
+      "reward_std": 0.24327749805524945,
+      "rewards/accuracy_reward": 0.24928572128992527,
+      "rewards/format_reward": 0.9728571444749832,
+      "step": 395
+    },
+    {
+      "completion_length": 347.8882158279419,
+      "epoch": 0.6184172383805199,
+      "grad_norm": 0.47993598719208436,
+      "kl": 0.2891845703125,
+      "learning_rate": 3.808470490337502e-06,
+      "loss": 0.0116,
+      "reward": 1.2317857250571251,
+      "reward_std": 0.2260462765581906,
+      "rewards/accuracy_reward": 0.2532142927404493,
+      "rewards/format_reward": 0.978571429848671,
+      "step": 400
+    },
+    {
+      "epoch": 0.6184172383805199,
+      "eval_completion_length": 318.71398162841797,
+      "eval_kl": 0.3338623046875,
+      "eval_loss": 0.013500670902431011,
+      "eval_reward": 1.2374999970197678,
+      "eval_reward_std": 0.24992241524159908,
+      "eval_rewards/accuracy_reward": 0.2607142888009548,
+      "eval_rewards/format_reward": 0.976785734295845,
+      "eval_runtime": 77.5724,
+      "eval_samples_per_second": 1.276,
+      "eval_steps_per_second": 0.026,
+      "step": 400
+    },
+    {
+      "completion_length": 340.0689334869385,
+      "epoch": 0.6261474538602764,
+      "grad_norm": 0.5074418639717388,
+      "kl": 0.3012939453125,
+      "learning_rate": 3.677636052371066e-06,
+      "loss": 0.0121,
+      "reward": 1.2257142968475818,
+      "reward_std": 0.23705143565312028,
+      "rewards/accuracy_reward": 0.2507142927730456,
+      "rewards/format_reward": 0.9750000014901161,
+      "step": 405
+    },
+    {
+      "completion_length": 334.34285869598386,
+      "epoch": 0.6338776693400329,
+      "grad_norm": 0.440989596288644,
+      "kl": 0.5120361328125,
+      "learning_rate": 3.547768137053523e-06,
+      "loss": 0.0205,
+      "reward": 1.2110714435577392,
+      "reward_std": 0.24922448974102734,
+      "rewards/accuracy_reward": 0.24642857799772172,
+      "rewards/format_reward": 0.9646428592503071,
+      "step": 410
+    },
+    {
+      "completion_length": 330.91714363098146,
+      "epoch": 0.6416078848197894,
+      "grad_norm": 0.26694024078543577,
+      "kl": 0.28255615234375,
+      "learning_rate": 3.4189616655158803e-06,
+      "loss": 0.0113,
+      "reward": 1.2403571613132953,
+      "reward_std": 0.2156613773666322,
+      "rewards/accuracy_reward": 0.2560714353807271,
+      "rewards/format_reward": 0.9842857152223587,
+      "step": 415
+    },
+    {
+      "completion_length": 373.41714534759524,
+      "epoch": 0.6493381002995459,
+      "grad_norm": 0.32331429774400394,
+      "kl": 0.262451171875,
+      "learning_rate": 3.291310783074405e-06,
+      "loss": 0.0105,
+      "reward": 1.26178572550416,
+      "reward_std": 0.22110692989081143,
+      "rewards/accuracy_reward": 0.2775000070221722,
+      "rewards/format_reward": 0.9842857152223587,
+      "step": 420
+    },
+    {
+      "completion_length": 410.3825017929077,
+      "epoch": 0.6570683157793024,
+      "grad_norm": 0.36969945354312556,
+      "kl": 0.34056396484375,
+      "learning_rate": 3.1649087904193142e-06,
+      "loss": 0.0136,
+      "reward": 1.2367857284843922,
+      "reward_std": 0.2502147083170712,
+      "rewards/accuracy_reward": 0.2603571502259001,
+      "rewards/format_reward": 0.976428572833538,
+      "step": 425
+    },
+    {
+      "completion_length": 401.2521469116211,
+      "epoch": 0.6647985312590589,
+      "grad_norm": 0.6482466478004806,
+      "kl": 0.38873291015625,
+      "learning_rate": 3.039848075420817e-06,
+      "loss": 0.0156,
+      "reward": 1.2332143016159534,
+      "reward_std": 0.2610732516273856,
+      "rewards/accuracy_reward": 0.2721428648335859,
+      "rewards/format_reward": 0.9610714308917523,
+      "step": 430
+    },
+    {
+      "completion_length": 416.10785961151123,
+      "epoch": 0.6725287467388154,
+      "grad_norm": 1.0368723402917392,
+      "kl": 0.44521484375,
+      "learning_rate": 2.916220045602349e-06,
+      "loss": 0.0178,
+      "reward": 1.2125000186264514,
+      "reward_std": 0.2809831270948052,
+      "rewards/accuracy_reward": 0.25500000710599124,
+      "rewards/format_reward": 0.9575000025331974,
+      "step": 435
+    },
+    {
+      "completion_length": 396.11535835266113,
+      "epoch": 0.6802589622185718,
+      "grad_norm": 0.4245630889591221,
+      "kl": 0.42137451171875,
+      "learning_rate": 2.794115061330336e-06,
+      "loss": 0.0169,
+      "reward": 1.2089285865426063,
+      "reward_std": 0.27289886865764856,
+      "rewards/accuracy_reward": 0.25464286408387127,
+      "rewards/format_reward": 0.954285717010498,
+      "step": 440
+    },
+    {
+      "completion_length": 405.51571731567384,
+      "epoch": 0.6879891776983283,
+      "grad_norm": 0.47571162730800487,
+      "kl": 0.45782470703125,
+      "learning_rate": 2.6736223697693387e-06,
+      "loss": 0.0183,
+      "reward": 1.21464286968112,
+      "reward_std": 0.2741479127667844,
+      "rewards/accuracy_reward": 0.2639285788871348,
+      "rewards/format_reward": 0.9507142886519432,
+      "step": 445
+    },
+    {
+      "completion_length": 378.1635715484619,
+      "epoch": 0.6957193931780848,
+      "grad_norm": 0.43896790616629155,
+      "kl": 0.4071044921875,
+      "learning_rate": 2.554830039650834e-06,
+      "loss": 0.0163,
+      "reward": 1.202500019222498,
+      "reward_std": 0.2554827404208481,
+      "rewards/accuracy_reward": 0.24857143468689175,
+      "rewards/format_reward": 0.9539285741746426,
+      "step": 450
+    },
+    {
+      "completion_length": 376.27357387542725,
+      "epoch": 0.7034496086578413,
+      "grad_norm": 0.6375781082839199,
+      "kl": 0.40799560546875,
+      "learning_rate": 2.4378248969033275e-06,
+      "loss": 0.0163,
+      "reward": 1.2100000150501729,
+      "reward_std": 0.2629528701305389,
+      "rewards/accuracy_reward": 0.24857143517583608,
+      "rewards/format_reward": 0.9614285737276077,
+      "step": 455
+    },
+    {
+      "completion_length": 400.56857414245604,
+      "epoch": 0.7111798241375978,
+      "grad_norm": 0.5893322722521851,
+      "kl": 0.46832275390625,
+      "learning_rate": 2.32269246119083e-06,
+      "loss": 0.0187,
+      "reward": 1.1839285910129547,
+      "reward_std": 0.2871158769354224,
+      "rewards/accuracy_reward": 0.24214286401402205,
+      "rewards/format_reward": 0.9417857177555561,
+      "step": 460
+    },
+    {
+      "completion_length": 426.16643199920657,
+      "epoch": 0.7189100396173543,
+      "grad_norm": 0.7285531480050851,
+      "kl": 0.45289306640625,
+      "learning_rate": 2.2095168834060982e-06,
+      "loss": 0.0181,
+      "reward": 1.213928584754467,
+      "reward_std": 0.27019564192742107,
+      "rewards/accuracy_reward": 0.2635714357485995,
+      "rewards/format_reward": 0.9503571458160878,
+      "step": 465
+    },
+    {
+      "completion_length": 426.24428634643556,
+      "epoch": 0.7266402550971108,
+      "grad_norm": 0.3644894126467112,
+      "kl": 0.44105224609375,
+      "learning_rate": 2.0983808841642996e-06,
+      "loss": 0.0176,
+      "reward": 1.227142871916294,
+      "reward_std": 0.2584091882221401,
+      "rewards/accuracy_reward": 0.2635714358650148,
+      "rewards/format_reward": 0.9635714307427407,
+      "step": 470
+    },
+    {
+      "completion_length": 398.2003578186035,
+      "epoch": 0.7343704705768673,
+      "grad_norm": 0.40238625933610955,
+      "kl": 0.339495849609375,
+      "learning_rate": 1.989365693342103e-06,
+      "loss": 0.0136,
+      "reward": 1.2378571562469005,
+      "reward_std": 0.24942016880959272,
+      "rewards/accuracy_reward": 0.2696428646799177,
+      "rewards/format_reward": 0.968214287608862,
+      "step": 475
+    },
+    {
+      "completion_length": 417.70250301361085,
+      "epoch": 0.7421006860566238,
+      "grad_norm": 0.4343672927770361,
+      "kl": 0.293597412109375,
+      "learning_rate": 1.8825509907063328e-06,
+      "loss": 0.0117,
+      "reward": 1.245714296400547,
+      "reward_std": 0.23920459682121872,
+      "rewards/accuracy_reward": 0.2710714365821332,
+      "rewards/format_reward": 0.9746428586542606,
+      "step": 480
+    },
+    {
+      "completion_length": 402.52250442504885,
+      "epoch": 0.7498309015363803,
+      "grad_norm": 0.5328436426470834,
+      "kl": 0.333837890625,
+      "learning_rate": 1.7780148476756148e-06,
+      "loss": 0.0134,
+      "reward": 1.2360714450478554,
+      "reward_std": 0.2392024408094585,
+      "rewards/accuracy_reward": 0.2632142927031964,
+      "rewards/format_reward": 0.9728571444749832,
+      "step": 485
+    },
+    {
+      "completion_length": 418.83428859710693,
+      "epoch": 0.7575611170161368,
+      "grad_norm": 0.3887859392004243,
+      "kl": 0.3875732421875,
+      "learning_rate": 1.6758336702575773e-06,
+      "loss": 0.0155,
+      "reward": 1.2235714450478554,
+      "reward_std": 0.26047459337860346,
+      "rewards/accuracy_reward": 0.25464286440983414,
+      "rewards/format_reward": 0.9689285732805729,
+      "step": 490
+    },
+    {
+      "completion_length": 418.6089317321777,
+      "epoch": 0.7652913324958933,
+      "grad_norm": 0.5048781836979007,
+      "kl": 0.3671630859375,
+      "learning_rate": 1.5760821432032953e-06,
+      "loss": 0.0147,
+      "reward": 1.2132142998278141,
+      "reward_std": 0.24158032452687622,
+      "rewards/accuracy_reward": 0.24642857771832496,
+      "rewards/format_reward": 0.96678571626544,
+      "step": 495
+    },
+    {
+      "completion_length": 401.3010726928711,
+      "epoch": 0.7730215479756498,
+      "grad_norm": 0.4607265236386812,
+      "kl": 0.4398681640625,
+      "learning_rate": 1.4788331754198316e-06,
+      "loss": 0.0176,
+      "reward": 1.2085714474320413,
+      "reward_std": 0.28208895614370705,
+      "rewards/accuracy_reward": 0.2539285784587264,
+      "rewards/format_reward": 0.9546428598463536,
+      "step": 500
+    },
+    {
+      "epoch": 0.7730215479756498,
+      "eval_completion_length": 354.90320205688477,
+      "eval_kl": 0.45703125,
+      "eval_loss": 0.018093252554535866,
+      "eval_reward": 1.226785734295845,
+      "eval_reward_std": 0.26129008270800114,
+      "eval_rewards/accuracy_reward": 0.2696428634226322,
+      "eval_rewards/format_reward": 0.9571428745985031,
+      "eval_runtime": 110.1034,
+      "eval_samples_per_second": 0.899,
+      "eval_steps_per_second": 0.018,
+      "step": 500
+    },
+    {
+      "completion_length": 402.9871459960938,
+      "epoch": 0.7807517634554063,
+      "grad_norm": 0.520985901336068,
+      "kl": 0.4232177734375,
+      "learning_rate": 1.384157846680737e-06,
+      "loss": 0.0169,
+      "reward": 1.2203571587800979,
+      "reward_std": 0.24645365020260215,
+      "rewards/accuracy_reward": 0.2632142924237996,
+      "rewards/format_reward": 0.9571428596973419,
+      "step": 505
+    },
+    {
+      "completion_length": 407.93928661346433,
+      "epoch": 0.7884819789351628,
+      "grad_norm": 0.3805648616018147,
+      "kl": 0.384832763671875,
+      "learning_rate": 1.2921253556734654e-06,
+      "loss": 0.0154,
+      "reward": 1.217857153713703,
+      "reward_std": 0.26537872953340413,
+      "rewards/accuracy_reward": 0.25535714966244993,
+      "rewards/format_reward": 0.9625000022351742,
+      "step": 510
+    },
+    {
+      "completion_length": 388.97393188476565,
+      "epoch": 0.7962121944149193,
+      "grad_norm": 1.1499390375989664,
+      "kl": 0.318841552734375,
+      "learning_rate": 1.202802969421708e-06,
+      "loss": 0.0128,
+      "reward": 1.198214303702116,
+      "reward_std": 0.23692603250965477,
+      "rewards/accuracy_reward": 0.23464286390226335,
+      "rewards/format_reward": 0.9635714307427407,
+      "step": 515
+    },
+    {
+      "completion_length": 383.35643138885496,
+      "epoch": 0.8039424098946758,
+      "grad_norm": 0.4367989993018896,
+      "kl": 0.34908447265625,
+      "learning_rate": 1.1162559741195733e-06,
+      "loss": 0.014,
+      "reward": 1.245357161015272,
+      "reward_std": 0.2558293900452554,
+      "rewards/accuracy_reward": 0.27678572088479997,
+      "rewards/format_reward": 0.9685714304447174,
+      "step": 520
+    },
+    {
+      "completion_length": 385.04857444763184,
+      "epoch": 0.8116726253744323,
+      "grad_norm": 0.6398289536823912,
+      "kl": 0.329840087890625,
+      "learning_rate": 1.0325476274135665e-06,
+      "loss": 0.0132,
+      "reward": 1.2332143023610116,
+      "reward_std": 0.23316460549831391,
+      "rewards/accuracy_reward": 0.2617857217788696,
+      "rewards/format_reward": 0.9714285731315613,
+      "step": 525
+    },
+    {
+      "completion_length": 385.87678928375243,
+      "epoch": 0.8194028408541888,
+      "grad_norm": 0.3695626628673057,
+      "kl": 0.39681396484375,
+      "learning_rate": 9.517391121672537e-07,
+      "loss": 0.0159,
+      "reward": 1.2292857207357883,
+      "reward_std": 0.2500005507841706,
+      "rewards/accuracy_reward": 0.26821429268457,
+      "rewards/format_reward": 0.9610714308917523,
+      "step": 530
+    },
+    {
+      "completion_length": 370.4653594970703,
+      "epoch": 0.8271330563339453,
+      "grad_norm": 0.6854915615610104,
+      "kl": 0.38438720703125,
+      "learning_rate": 8.738894917423918e-07,
+      "loss": 0.0154,
+      "reward": 1.2175000138580798,
+      "reward_std": 0.24949982650578023,
+      "rewards/accuracy_reward": 0.2521428640931845,
+      "rewards/format_reward": 0.9653571449220181,
+      "step": 535
+    },
+    {
+      "completion_length": 373.39250297546386,
+      "epoch": 0.8348632718137018,
+      "grad_norm": 0.3428641792816048,
+      "kl": 0.3291259765625,
+      "learning_rate": 7.990556668292021e-07,
+      "loss": 0.0132,
+      "reward": 1.2228571593761444,
+      "reward_std": 0.2329377906396985,
+      "rewards/accuracy_reward": 0.2507142930058762,
+      "rewards/format_reward": 0.9721428588032722,
+      "step": 540
+    },
+    {
+      "completion_length": 390.6789312362671,
+      "epoch": 0.8425934872934583,
+      "grad_norm": 0.8781910108571961,
+      "kl": 0.3635009765625,
+      "learning_rate": 7.272923338573723e-07,
+      "loss": 0.0145,
+      "reward": 1.2246428735554218,
+      "reward_std": 0.24744346737861633,
+      "rewards/accuracy_reward": 0.2567857220768929,
+      "rewards/format_reward": 0.9678571447730064,
+      "step": 545
+    },
+    {
+      "completion_length": 386.65393047332765,
+      "epoch": 0.8503237027732148,
+      "grad_norm": 0.374646921874402,
+      "kl": 0.305267333984375,
+      "learning_rate": 6.586519450181488e-07,
+      "loss": 0.0122,
+      "reward": 1.242500015348196,
+      "reward_std": 0.24353330470621587,
+      "rewards/accuracy_reward": 0.26964286335278304,
+      "rewards/format_reward": 0.9728571444749832,
+      "step": 550
+    },
+    {
+      "completion_length": 384.5810733795166,
+      "epoch": 0.8580539182529713,
+      "grad_norm": 0.5549528232027495,
+      "kl": 0.312255859375,
+      "learning_rate": 5.931846699267558e-07,
+      "loss": 0.0125,
+      "reward": 1.247857154905796,
+      "reward_std": 0.25878492267802355,
+      "rewards/accuracy_reward": 0.2767857232829556,
+      "rewards/format_reward": 0.9710714302957058,
+      "step": 555
+    },
+    {
+      "completion_length": 386.3992879867554,
+      "epoch": 0.8657841337327278,
+      "grad_norm": 0.4278411326159335,
+      "kl": 0.34853515625,
+      "learning_rate": 5.309383589531708e-07,
+      "loss": 0.0139,
+      "reward": 1.2271428763866425,
+      "reward_std": 0.23170938733965157,
+      "rewards/accuracy_reward": 0.2575000062817708,
+      "rewards/format_reward": 0.9696428589522839,
+      "step": 560
+    },
+    {
+      "completion_length": 391.38536109924314,
+      "epoch": 0.8735143492124843,
+      "grad_norm": 0.4003970163311169,
+      "kl": 0.37874755859375,
+      "learning_rate": 4.7195850824803823e-07,
+      "loss": 0.0152,
+      "reward": 1.2192857332527638,
+      "reward_std": 0.25189957171678545,
+      "rewards/accuracy_reward": 0.2528571498580277,
+      "rewards/format_reward": 0.9664285734295845,
+      "step": 565
+    },
+    {
+      "completion_length": 384.1317882537842,
+      "epoch": 0.8812445646922408,
+      "grad_norm": 0.4111180386794067,
+      "kl": 0.358599853515625,
+      "learning_rate": 4.1628822648929255e-07,
+      "loss": 0.0143,
+      "reward": 1.2210714399814606,
+      "reward_std": 0.24504473190754653,
+      "rewards/accuracy_reward": 0.25392857876140623,
+      "rewards/format_reward": 0.9671428591012955,
+      "step": 570
+    },
+    {
+      "completion_length": 400.0792875289917,
+      "epoch": 0.8889747801719973,
+      "grad_norm": 0.44019676078400016,
+      "kl": 0.317877197265625,
+      "learning_rate": 3.6396820337381236e-07,
+      "loss": 0.0127,
+      "reward": 1.2107143074274063,
+      "reward_std": 0.23559978352859617,
+      "rewards/accuracy_reward": 0.24178572117816657,
+      "rewards/format_reward": 0.9689285732805729,
+      "step": 575
+    },
+    {
+      "completion_length": 391.342502784729,
+      "epoch": 0.8967049956517538,
+      "grad_norm": 0.46727020273244035,
+      "kl": 0.312957763671875,
+      "learning_rate": 3.150366798771043e-07,
+      "loss": 0.0125,
+      "reward": 1.2439285896718502,
+      "reward_std": 0.2462494120001793,
+      "rewards/accuracy_reward": 0.2714285790920258,
+      "rewards/format_reward": 0.9725000016391278,
+      "step": 580
+    },
+    {
+      "completion_length": 372.30536060333253,
+      "epoch": 0.9044352111315103,
+      "grad_norm": 0.5209042194427927,
+      "kl": 0.319873046875,
+      "learning_rate": 2.6952942030277486e-07,
+      "loss": 0.0128,
+      "reward": 1.2671428687870503,
+      "reward_std": 0.26302206302061676,
+      "rewards/accuracy_reward": 0.2996428650803864,
+      "rewards/format_reward": 0.9675000019371509,
+      "step": 585
+    },
+    {
+      "completion_length": 394.72285861968993,
+      "epoch": 0.9121654266112668,
+      "grad_norm": 0.4137939679586483,
+      "kl": 0.33170166015625,
+      "learning_rate": 2.274796861422246e-07,
+      "loss": 0.0133,
+      "reward": 1.2225000075995922,
+      "reward_std": 0.24834517342969775,
+      "rewards/accuracy_reward": 0.25821429330389944,
+      "rewards/format_reward": 0.9642857164144516,
+      "step": 590
+    },
+    {
+      "completion_length": 402.35428485870364,
+      "epoch": 0.9198956420910233,
+      "grad_norm": 0.412787643650894,
+      "kl": 0.37198486328125,
+      "learning_rate": 1.88918211763639e-07,
+      "loss": 0.0149,
+      "reward": 1.2421428732573987,
+      "reward_std": 0.2519141098484397,
+      "rewards/accuracy_reward": 0.2735714362002909,
+      "rewards/format_reward": 0.9685714304447174,
+      "step": 595
+    },
+    {
+      "completion_length": 381.8421464920044,
+      "epoch": 0.9276258575707798,
+      "grad_norm": 0.4865523678871763,
+      "kl": 0.323974609375,
+      "learning_rate": 1.5387318194809098e-07,
+      "loss": 0.013,
+      "reward": 1.2353571593761443,
+      "reward_std": 0.22969860946759582,
+      "rewards/accuracy_reward": 0.2639285792130977,
+      "rewards/format_reward": 0.9714285731315613,
+      "step": 600
+    },
+    {
+      "epoch": 0.9276258575707798,
+      "eval_completion_length": 353.1224174499512,
+      "eval_kl": 0.30078125,
+      "eval_loss": 0.011807209812104702,
+      "eval_reward": 1.2553571313619614,
+      "eval_reward_std": 0.22612838447093964,
+      "eval_rewards/accuracy_reward": 0.2821428645402193,
+      "eval_rewards/format_reward": 0.973214291036129,
+      "eval_runtime": 111.1459,
+      "eval_samples_per_second": 0.891,
+      "eval_steps_per_second": 0.018,
+      "step": 600
+    },
+    {
+      "completion_length": 404.2853603363037,
+      "epoch": 0.9353560730505363,
+      "grad_norm": 0.5171782261479181,
+      "kl": 0.34097900390625,
+      "learning_rate": 1.223702112891345e-07,
+      "loss": 0.0136,
+      "reward": 1.2089285850524902,
+      "reward_std": 0.24284306410700082,
+      "rewards/accuracy_reward": 0.2428571501513943,
+      "rewards/format_reward": 0.966071430593729,
+      "step": 605
+    },
+    {
+      "completion_length": 397.3825029373169,
+      "epoch": 0.9430862885302927,
+      "grad_norm": 0.7320475164079379,
+      "kl": 0.3185546875,
+      "learning_rate": 9.443232547096448e-08,
+      "loss": 0.0127,
+      "reward": 1.256428588926792,
+      "reward_std": 0.26404928769916297,
+      "rewards/accuracy_reward": 0.2846428649034351,
+      "rewards/format_reward": 0.9717857159674168,
+      "step": 610
+    },
+    {
+      "completion_length": 407.19285888671874,
+      "epoch": 0.9508165040100492,
+      "grad_norm": 1.2171362297470418,
+      "kl": 0.36600341796875,
+      "learning_rate": 7.007994443882616e-08,
+      "loss": 0.0146,
+      "reward": 1.210714304447174,
+      "reward_std": 0.26716623855754734,
+      "rewards/accuracy_reward": 0.24892857847735286,
+      "rewards/format_reward": 0.9617857165634632,
+      "step": 615
+    },
+    {
+      "completion_length": 395.53178749084475,
+      "epoch": 0.9585467194898057,
+      "grad_norm": 0.36444035142958237,
+      "kl": 0.3221923828125,
+      "learning_rate": 4.933086747397231e-08,
+      "loss": 0.0129,
+      "reward": 1.2253571592271328,
+      "reward_std": 0.23916244860738517,
+      "rewards/accuracy_reward": 0.2553571488009766,
+      "rewards/format_reward": 0.9700000017881394,
+      "step": 620
+    },
+    {
+      "completion_length": 393.6335744857788,
+      "epoch": 0.9662769349695622,
+      "grad_norm": 0.3989424097774144,
+      "kl": 0.3381103515625,
+      "learning_rate": 3.220026018407541e-08,
+      "loss": 0.0135,
+      "reward": 1.2310714446008206,
+      "reward_std": 0.23451657630503178,
+      "rewards/accuracy_reward": 0.2621428639627993,
+      "rewards/format_reward": 0.9689285732805729,
+      "step": 625
+    },
+    {
+      "completion_length": 396.7300016403198,
+      "epoch": 0.9740071504493187,
+      "grad_norm": 0.7711804308584844,
+      "kl": 0.33509521484375,
+      "learning_rate": 1.8700643418607112e-08,
+      "loss": 0.0134,
+      "reward": 1.250000011920929,
+      "reward_std": 0.26608177088201046,
+      "rewards/accuracy_reward": 0.2835714362096041,
+      "rewards/format_reward": 0.9664285734295845,
+      "step": 630
+    },
+    {
+      "completion_length": 392.350359916687,
+      "epoch": 0.9817373659290752,
+      "grad_norm": 0.42013697611704715,
+      "kl": 0.327276611328125,
+      "learning_rate": 8.841884117284372e-09,
+      "loss": 0.0131,
+      "reward": 1.241785728931427,
+      "reward_std": 0.24906472209841013,
+      "rewards/accuracy_reward": 0.270714293140918,
+      "rewards/format_reward": 0.9710714302957058,
+      "step": 635
+    },
+    {
+      "completion_length": 379.29678897857667,
+      "epoch": 0.9894675814088317,
+      "grad_norm": 0.39572354868452714,
+      "kl": 0.298919677734375,
+      "learning_rate": 2.6311880982721195e-09,
+      "loss": 0.012,
+      "reward": 1.248571439087391,
+      "reward_std": 0.25294895516708493,
+      "rewards/accuracy_reward": 0.2753571497276425,
+      "rewards/format_reward": 0.9732142873108387,
+      "step": 640
+    },
+    {
+      "completion_length": 391.0450023651123,
+      "epoch": 0.9971977968885882,
+      "grad_norm": 0.4471955331682869,
+      "kl": 0.334100341796875,
+      "learning_rate": 7.30947914140412e-11,
+      "loss": 0.0134,
+      "reward": 1.2375000178813935,
+      "reward_std": 0.24879253581166266,
+      "rewards/accuracy_reward": 0.26464286535046994,
+      "rewards/format_reward": 0.9728571444749832,
+      "step": 645
+    },
+    {
+      "completion_length": 370.5642910003662,
+      "epoch": 0.9987438399845395,
+      "kl": 0.784271240234375,
+      "reward": 1.2250000201165676,
+      "reward_std": 0.259319587610662,
+      "rewards/accuracy_reward": 0.25714286393485963,
+      "rewards/format_reward": 0.9678571447730064,
+      "step": 646,
+      "total_flos": 0.0,
+      "train_loss": 0.01271514138510042,
+      "train_runtime": 147461.7668,
+      "train_samples_per_second": 0.491,
+      "train_steps_per_second": 0.004
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 646,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 5,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcb5bdda4f17eb1fd0d6349eced4d9ec471f751107ad5bf83f361ddac1e00393
+size 7544