Add files using upload-large-folder tool

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 checkpoints/Qwen2.5-14B/babylm_shuffle_nondeterministic_10M_seed0/runs/checkpoint-1934/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/Qwen2.5-14B/babylm_shuffle_nondeterministic_10M_seed0/runs/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/Qwen2.5-14B/babylm_shuffle_nondeterministic_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/blobs/9b693615965d8548eab4d1dc6bb578aa063e8656e2b9d25125ad5b8c7f59565f filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen2.5-14B/babylm_hop_words4_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/blobs/9b693615965d8548eab4d1dc6bb578aa063e8656e2b9d25125ad5b8c7f59565f filter=lfs diff=lfs merge=lfs -text

checkpoints/Qwen2.5-14B/babylm_hop_control_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/snapshots/97e1e76335b7017d8f67c08a19d103c0504298c9/model-00008-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b693615965d8548eab4d1dc6bb578aa063e8656e2b9d25125ad5b8c7f59565f
+size 1698724408

checkpoints/Qwen2.5-14B/babylm_hop_words4_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/blobs/9b693615965d8548eab4d1dc6bb578aa063e8656e2b9d25125ad5b8c7f59565f

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b693615965d8548eab4d1dc6bb578aa063e8656e2b9d25125ad5b8c7f59565f
+size 1698724408

checkpoints/Qwen2.5-14B/babylm_hop_words4_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/snapshots/97e1e76335b7017d8f67c08a19d103c0504298c9/model-00008-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b693615965d8548eab4d1dc6bb578aa063e8656e2b9d25125ad5b8c7f59565f
+size 1698724408

checkpoints/Qwen2.5-14B/babylm_shuffle_control_10M_seed0/runs/checkpoint-500/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_control_10M_seed0/runs/checkpoint-500/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-14B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 13824,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 48,
+  "model_type": "qwen2",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_control_10M_seed0/runs/checkpoint-500/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_control_10M_seed0/runs/checkpoint-500/trainer_state.json

@@ -0,0 +1,3613 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5167958656330749,
+  "eval_steps": 50,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0010335917312661498,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6607,
+      "step": 1
+    },
+    {
+      "epoch": 0.0020671834625322996,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6461,
+      "step": 2
+    },
+    {
+      "epoch": 0.0031007751937984496,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6883,
+      "step": 3
+    },
+    {
+      "epoch": 0.004134366925064599,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6281,
+      "step": 4
+    },
+    {
+      "epoch": 0.00516795865633075,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6642,
+      "step": 5
+    },
+    {
+      "epoch": 0.006201550387596899,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6784,
+      "step": 6
+    },
+    {
+      "epoch": 0.007235142118863049,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6831,
+      "step": 7
+    },
+    {
+      "epoch": 0.008268733850129198,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6472,
+      "step": 8
+    },
+    {
+      "epoch": 0.009302325581395349,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6772,
+      "step": 9
+    },
+    {
+      "epoch": 0.0103359173126615,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6972,
+      "step": 10
+    },
+    {
+      "epoch": 0.011369509043927648,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6912,
+      "step": 11
+    },
+    {
+      "epoch": 0.012403100775193798,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.667,
+      "step": 12
+    },
+    {
+      "epoch": 0.013436692506459949,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6719,
+      "step": 13
+    },
+    {
+      "epoch": 0.014470284237726097,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6269,
+      "step": 14
+    },
+    {
+      "epoch": 0.015503875968992248,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6598,
+      "step": 15
+    },
+    {
+      "epoch": 0.016537467700258397,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.6819,
+      "step": 16
+    },
+    {
+      "epoch": 0.01757105943152455,
+      "grad_norm": 2.897101640701294,
+      "learning_rate": 2.5773195876288662e-08,
+      "loss": 1.6717,
+      "step": 17
+    },
+    {
+      "epoch": 0.018604651162790697,
+      "grad_norm": 3.3035874366760254,
+      "learning_rate": 5.1546391752577325e-08,
+      "loss": 1.7092,
+      "step": 18
+    },
+    {
+      "epoch": 0.019638242894056846,
+      "grad_norm": 2.7663118839263916,
+      "learning_rate": 7.731958762886598e-08,
+      "loss": 1.6714,
+      "step": 19
+    },
+    {
+      "epoch": 0.020671834625323,
+      "grad_norm": 2.696624279022217,
+      "learning_rate": 1.0309278350515465e-07,
+      "loss": 1.6648,
+      "step": 20
+    },
+    {
+      "epoch": 0.021705426356589147,
+      "grad_norm": 2.572075366973877,
+      "learning_rate": 1.288659793814433e-07,
+      "loss": 1.649,
+      "step": 21
+    },
+    {
+      "epoch": 0.022739018087855296,
+      "grad_norm": 3.8518550395965576,
+      "learning_rate": 1.5463917525773197e-07,
+      "loss": 1.6555,
+      "step": 22
+    },
+    {
+      "epoch": 0.023772609819121448,
+      "grad_norm": 2.9744110107421875,
+      "learning_rate": 1.804123711340206e-07,
+      "loss": 1.6683,
+      "step": 23
+    },
+    {
+      "epoch": 0.024806201550387597,
+      "grad_norm": 3.381155014038086,
+      "learning_rate": 2.061855670103093e-07,
+      "loss": 1.6864,
+      "step": 24
+    },
+    {
+      "epoch": 0.025839793281653745,
+      "grad_norm": 3.5349085330963135,
+      "learning_rate": 2.3195876288659797e-07,
+      "loss": 1.6787,
+      "step": 25
+    },
+    {
+      "epoch": 0.026873385012919897,
+      "grad_norm": 2.8829333782196045,
+      "learning_rate": 2.577319587628866e-07,
+      "loss": 1.6425,
+      "step": 26
+    },
+    {
+      "epoch": 0.027906976744186046,
+      "grad_norm": 2.7831168174743652,
+      "learning_rate": 2.8350515463917527e-07,
+      "loss": 1.6556,
+      "step": 27
+    },
+    {
+      "epoch": 0.028940568475452195,
+      "grad_norm": 3.226625919342041,
+      "learning_rate": 3.0927835051546394e-07,
+      "loss": 1.6755,
+      "step": 28
+    },
+    {
+      "epoch": 0.029974160206718347,
+      "grad_norm": 2.7015786170959473,
+      "learning_rate": 3.350515463917526e-07,
+      "loss": 1.6677,
+      "step": 29
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 2.572233200073242,
+      "learning_rate": 3.608247422680412e-07,
+      "loss": 1.6925,
+      "step": 30
+    },
+    {
+      "epoch": 0.03204134366925065,
+      "grad_norm": 2.339751958847046,
+      "learning_rate": 3.8659793814432993e-07,
+      "loss": 1.6202,
+      "step": 31
+    },
+    {
+      "epoch": 0.03307493540051679,
+      "grad_norm": 2.538344621658325,
+      "learning_rate": 4.123711340206186e-07,
+      "loss": 1.634,
+      "step": 32
+    },
+    {
+      "epoch": 0.034108527131782945,
+      "grad_norm": 1.9859461784362793,
+      "learning_rate": 4.381443298969072e-07,
+      "loss": 1.6472,
+      "step": 33
+    },
+    {
+      "epoch": 0.0351421188630491,
+      "grad_norm": 2.0745434761047363,
+      "learning_rate": 4.6391752577319593e-07,
+      "loss": 1.662,
+      "step": 34
+    },
+    {
+      "epoch": 0.03617571059431524,
+      "grad_norm": 1.5810471773147583,
+      "learning_rate": 4.896907216494846e-07,
+      "loss": 1.6466,
+      "step": 35
+    },
+    {
+      "epoch": 0.037209302325581395,
+      "grad_norm": 1.4383913278579712,
+      "learning_rate": 5.154639175257732e-07,
+      "loss": 1.6553,
+      "step": 36
+    },
+    {
+      "epoch": 0.03824289405684755,
+      "grad_norm": 2.5648560523986816,
+      "learning_rate": 5.412371134020619e-07,
+      "loss": 1.638,
+      "step": 37
+    },
+    {
+      "epoch": 0.03927648578811369,
+      "grad_norm": 1.772307276725769,
+      "learning_rate": 5.670103092783505e-07,
+      "loss": 1.5873,
+      "step": 38
+    },
+    {
+      "epoch": 0.040310077519379844,
+      "grad_norm": 1.5337371826171875,
+      "learning_rate": 5.927835051546392e-07,
+      "loss": 1.6429,
+      "step": 39
+    },
+    {
+      "epoch": 0.041343669250646,
+      "grad_norm": 1.5859408378601074,
+      "learning_rate": 6.185567010309279e-07,
+      "loss": 1.6351,
+      "step": 40
+    },
+    {
+      "epoch": 0.04237726098191214,
+      "grad_norm": 1.5069128274917603,
+      "learning_rate": 6.443298969072165e-07,
+      "loss": 1.6293,
+      "step": 41
+    },
+    {
+      "epoch": 0.043410852713178294,
+      "grad_norm": 1.5274145603179932,
+      "learning_rate": 6.701030927835052e-07,
+      "loss": 1.6553,
+      "step": 42
+    },
+    {
+      "epoch": 0.044444444444444446,
+      "grad_norm": 2.2933266162872314,
+      "learning_rate": 6.958762886597939e-07,
+      "loss": 1.5855,
+      "step": 43
+    },
+    {
+      "epoch": 0.04547803617571059,
+      "grad_norm": 2.9122445583343506,
+      "learning_rate": 7.216494845360824e-07,
+      "loss": 1.6064,
+      "step": 44
+    },
+    {
+      "epoch": 0.046511627906976744,
+      "grad_norm": 1.6369024515151978,
+      "learning_rate": 7.474226804123711e-07,
+      "loss": 1.5924,
+      "step": 45
+    },
+    {
+      "epoch": 0.047545219638242896,
+      "grad_norm": 1.8904677629470825,
+      "learning_rate": 7.731958762886599e-07,
+      "loss": 1.5818,
+      "step": 46
+    },
+    {
+      "epoch": 0.04857881136950904,
+      "grad_norm": 2.2924325466156006,
+      "learning_rate": 7.989690721649485e-07,
+      "loss": 1.5867,
+      "step": 47
+    },
+    {
+      "epoch": 0.04961240310077519,
+      "grad_norm": 3.223383665084839,
+      "learning_rate": 8.247422680412372e-07,
+      "loss": 1.5756,
+      "step": 48
+    },
+    {
+      "epoch": 0.050645994832041345,
+      "grad_norm": 1.4551427364349365,
+      "learning_rate": 8.505154639175259e-07,
+      "loss": 1.589,
+      "step": 49
+    },
+    {
+      "epoch": 0.05167958656330749,
+      "grad_norm": 1.8233811855316162,
+      "learning_rate": 8.762886597938144e-07,
+      "loss": 1.5486,
+      "step": 50
+    },
+    {
+      "epoch": 0.05167958656330749,
+      "eval_loss": 1.570243239402771,
+      "eval_runtime": 53.2671,
+      "eval_samples_per_second": 18.773,
+      "eval_steps_per_second": 0.601,
+      "step": 50
+    },
+    {
+      "epoch": 0.05271317829457364,
+      "grad_norm": 2.7552506923675537,
+      "learning_rate": 9.020618556701031e-07,
+      "loss": 1.5484,
+      "step": 51
+    },
+    {
+      "epoch": 0.053746770025839795,
+      "grad_norm": 1.6123155355453491,
+      "learning_rate": 9.278350515463919e-07,
+      "loss": 1.5412,
+      "step": 52
+    },
+    {
+      "epoch": 0.05478036175710594,
+      "grad_norm": 1.4314796924591064,
+      "learning_rate": 9.536082474226805e-07,
+      "loss": 1.5393,
+      "step": 53
+    },
+    {
+      "epoch": 0.05581395348837209,
+      "grad_norm": 1.6784818172454834,
+      "learning_rate": 9.793814432989692e-07,
+      "loss": 1.5458,
+      "step": 54
+    },
+    {
+      "epoch": 0.056847545219638244,
+      "grad_norm": 1.7028733491897583,
+      "learning_rate": 1.005154639175258e-06,
+      "loss": 1.4995,
+      "step": 55
+    },
+    {
+      "epoch": 0.05788113695090439,
+      "grad_norm": 1.6766446828842163,
+      "learning_rate": 1.0309278350515464e-06,
+      "loss": 1.505,
+      "step": 56
+    },
+    {
+      "epoch": 0.05891472868217054,
+      "grad_norm": 1.7980971336364746,
+      "learning_rate": 1.0567010309278351e-06,
+      "loss": 1.4519,
+      "step": 57
+    },
+    {
+      "epoch": 0.059948320413436694,
+      "grad_norm": 1.8206804990768433,
+      "learning_rate": 1.0824742268041239e-06,
+      "loss": 1.4577,
+      "step": 58
+    },
+    {
+      "epoch": 0.06098191214470284,
+      "grad_norm": 1.6603621244430542,
+      "learning_rate": 1.1082474226804124e-06,
+      "loss": 1.4541,
+      "step": 59
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 1.7833083868026733,
+      "learning_rate": 1.134020618556701e-06,
+      "loss": 1.4556,
+      "step": 60
+    },
+    {
+      "epoch": 0.06304909560723514,
+      "grad_norm": 1.9101687669754028,
+      "learning_rate": 1.1597938144329898e-06,
+      "loss": 1.4173,
+      "step": 61
+    },
+    {
+      "epoch": 0.0640826873385013,
+      "grad_norm": 1.8884676694869995,
+      "learning_rate": 1.1855670103092783e-06,
+      "loss": 1.4172,
+      "step": 62
+    },
+    {
+      "epoch": 0.06511627906976744,
+      "grad_norm": 1.963536262512207,
+      "learning_rate": 1.211340206185567e-06,
+      "loss": 1.4207,
+      "step": 63
+    },
+    {
+      "epoch": 0.06614987080103359,
+      "grad_norm": 2.2165422439575195,
+      "learning_rate": 1.2371134020618557e-06,
+      "loss": 1.4136,
+      "step": 64
+    },
+    {
+      "epoch": 0.06718346253229975,
+      "grad_norm": 1.333492398262024,
+      "learning_rate": 1.2628865979381445e-06,
+      "loss": 1.4001,
+      "step": 65
+    },
+    {
+      "epoch": 0.06821705426356589,
+      "grad_norm": 1.4832924604415894,
+      "learning_rate": 1.288659793814433e-06,
+      "loss": 1.3924,
+      "step": 66
+    },
+    {
+      "epoch": 0.06925064599483204,
+      "grad_norm": 1.7485769987106323,
+      "learning_rate": 1.314432989690722e-06,
+      "loss": 1.3667,
+      "step": 67
+    },
+    {
+      "epoch": 0.0702842377260982,
+      "grad_norm": 1.5255411863327026,
+      "learning_rate": 1.3402061855670104e-06,
+      "loss": 1.3803,
+      "step": 68
+    },
+    {
+      "epoch": 0.07131782945736434,
+      "grad_norm": 1.6617186069488525,
+      "learning_rate": 1.365979381443299e-06,
+      "loss": 1.3814,
+      "step": 69
+    },
+    {
+      "epoch": 0.07235142118863049,
+      "grad_norm": 1.7209995985031128,
+      "learning_rate": 1.3917525773195878e-06,
+      "loss": 1.4081,
+      "step": 70
+    },
+    {
+      "epoch": 0.07338501291989664,
+      "grad_norm": 1.6906920671463013,
+      "learning_rate": 1.4175257731958764e-06,
+      "loss": 1.3755,
+      "step": 71
+    },
+    {
+      "epoch": 0.07441860465116279,
+      "grad_norm": 1.5806481838226318,
+      "learning_rate": 1.4432989690721649e-06,
+      "loss": 1.3394,
+      "step": 72
+    },
+    {
+      "epoch": 0.07545219638242893,
+      "grad_norm": 1.6998566389083862,
+      "learning_rate": 1.4690721649484538e-06,
+      "loss": 1.3773,
+      "step": 73
+    },
+    {
+      "epoch": 0.0764857881136951,
+      "grad_norm": 1.4243597984313965,
+      "learning_rate": 1.4948453608247423e-06,
+      "loss": 1.3232,
+      "step": 74
+    },
+    {
+      "epoch": 0.07751937984496124,
+      "grad_norm": 1.4643871784210205,
+      "learning_rate": 1.520618556701031e-06,
+      "loss": 1.312,
+      "step": 75
+    },
+    {
+      "epoch": 0.07855297157622738,
+      "grad_norm": 1.566022515296936,
+      "learning_rate": 1.5463917525773197e-06,
+      "loss": 1.3744,
+      "step": 76
+    },
+    {
+      "epoch": 0.07958656330749354,
+      "grad_norm": 1.6502660512924194,
+      "learning_rate": 1.5721649484536082e-06,
+      "loss": 1.2804,
+      "step": 77
+    },
+    {
+      "epoch": 0.08062015503875969,
+      "grad_norm": 1.4917631149291992,
+      "learning_rate": 1.597938144329897e-06,
+      "loss": 1.3368,
+      "step": 78
+    },
+    {
+      "epoch": 0.08165374677002583,
+      "grad_norm": 1.1608281135559082,
+      "learning_rate": 1.6237113402061857e-06,
+      "loss": 1.3036,
+      "step": 79
+    },
+    {
+      "epoch": 0.082687338501292,
+      "grad_norm": 1.4414565563201904,
+      "learning_rate": 1.6494845360824744e-06,
+      "loss": 1.3084,
+      "step": 80
+    },
+    {
+      "epoch": 0.08372093023255814,
+      "grad_norm": 1.8089519739151,
+      "learning_rate": 1.675257731958763e-06,
+      "loss": 1.3362,
+      "step": 81
+    },
+    {
+      "epoch": 0.08475452196382428,
+      "grad_norm": 1.835245966911316,
+      "learning_rate": 1.7010309278350518e-06,
+      "loss": 1.2728,
+      "step": 82
+    },
+    {
+      "epoch": 0.08578811369509044,
+      "grad_norm": 1.3364133834838867,
+      "learning_rate": 1.7268041237113403e-06,
+      "loss": 1.2804,
+      "step": 83
+    },
+    {
+      "epoch": 0.08682170542635659,
+      "grad_norm": 1.287848711013794,
+      "learning_rate": 1.7525773195876288e-06,
+      "loss": 1.3104,
+      "step": 84
+    },
+    {
+      "epoch": 0.08785529715762273,
+      "grad_norm": 1.3792903423309326,
+      "learning_rate": 1.7783505154639178e-06,
+      "loss": 1.2808,
+      "step": 85
+    },
+    {
+      "epoch": 0.08888888888888889,
+      "grad_norm": 1.4002866744995117,
+      "learning_rate": 1.8041237113402063e-06,
+      "loss": 1.2652,
+      "step": 86
+    },
+    {
+      "epoch": 0.08992248062015504,
+      "grad_norm": 1.423072099685669,
+      "learning_rate": 1.8298969072164948e-06,
+      "loss": 1.3136,
+      "step": 87
+    },
+    {
+      "epoch": 0.09095607235142118,
+      "grad_norm": 1.472794771194458,
+      "learning_rate": 1.8556701030927837e-06,
+      "loss": 1.3067,
+      "step": 88
+    },
+    {
+      "epoch": 0.09198966408268734,
+      "grad_norm": 1.3818236589431763,
+      "learning_rate": 1.8814432989690722e-06,
+      "loss": 1.2773,
+      "step": 89
+    },
+    {
+      "epoch": 0.09302325581395349,
+      "grad_norm": 1.799110770225525,
+      "learning_rate": 1.907216494845361e-06,
+      "loss": 1.2654,
+      "step": 90
+    },
+    {
+      "epoch": 0.09405684754521963,
+      "grad_norm": 1.6194965839385986,
+      "learning_rate": 1.9329896907216497e-06,
+      "loss": 1.2556,
+      "step": 91
+    },
+    {
+      "epoch": 0.09509043927648579,
+      "grad_norm": 1.6596431732177734,
+      "learning_rate": 1.9587628865979384e-06,
+      "loss": 1.2817,
+      "step": 92
+    },
+    {
+      "epoch": 0.09612403100775194,
+      "grad_norm": 1.6827796697616577,
+      "learning_rate": 1.9845360824742267e-06,
+      "loss": 1.2633,
+      "step": 93
+    },
+    {
+      "epoch": 0.09715762273901808,
+      "grad_norm": 1.3558834791183472,
+      "learning_rate": 2.010309278350516e-06,
+      "loss": 1.3066,
+      "step": 94
+    },
+    {
+      "epoch": 0.09819121447028424,
+      "grad_norm": 1.5518406629562378,
+      "learning_rate": 2.036082474226804e-06,
+      "loss": 1.2849,
+      "step": 95
+    },
+    {
+      "epoch": 0.09922480620155039,
+      "grad_norm": 1.3061556816101074,
+      "learning_rate": 2.061855670103093e-06,
+      "loss": 1.2521,
+      "step": 96
+    },
+    {
+      "epoch": 0.10025839793281653,
+      "grad_norm": 1.401341438293457,
+      "learning_rate": 2.0876288659793816e-06,
+      "loss": 1.2901,
+      "step": 97
+    },
+    {
+      "epoch": 0.10129198966408269,
+      "grad_norm": 1.5926023721694946,
+      "learning_rate": 2.1134020618556703e-06,
+      "loss": 1.2446,
+      "step": 98
+    },
+    {
+      "epoch": 0.10232558139534884,
+      "grad_norm": 1.6969091892242432,
+      "learning_rate": 2.139175257731959e-06,
+      "loss": 1.2243,
+      "step": 99
+    },
+    {
+      "epoch": 0.10335917312661498,
+      "grad_norm": 1.4960438013076782,
+      "learning_rate": 2.1649484536082477e-06,
+      "loss": 1.2166,
+      "step": 100
+    },
+    {
+      "epoch": 0.10335917312661498,
+      "eval_loss": 1.3024815320968628,
+      "eval_runtime": 52.9952,
+      "eval_samples_per_second": 18.87,
+      "eval_steps_per_second": 0.604,
+      "step": 100
+    },
+    {
+      "epoch": 0.10439276485788114,
+      "grad_norm": 1.7148933410644531,
+      "learning_rate": 2.1907216494845364e-06,
+      "loss": 1.2683,
+      "step": 101
+    },
+    {
+      "epoch": 0.10542635658914729,
+      "grad_norm": 1.396099328994751,
+      "learning_rate": 2.2164948453608247e-06,
+      "loss": 1.1986,
+      "step": 102
+    },
+    {
+      "epoch": 0.10645994832041343,
+      "grad_norm": 1.407954216003418,
+      "learning_rate": 2.242268041237114e-06,
+      "loss": 1.2169,
+      "step": 103
+    },
+    {
+      "epoch": 0.10749354005167959,
+      "grad_norm": 1.4110890626907349,
+      "learning_rate": 2.268041237113402e-06,
+      "loss": 1.222,
+      "step": 104
+    },
+    {
+      "epoch": 0.10852713178294573,
+      "grad_norm": 1.755106806755066,
+      "learning_rate": 2.293814432989691e-06,
+      "loss": 1.2718,
+      "step": 105
+    },
+    {
+      "epoch": 0.10956072351421188,
+      "grad_norm": 1.5914361476898193,
+      "learning_rate": 2.3195876288659796e-06,
+      "loss": 1.2251,
+      "step": 106
+    },
+    {
+      "epoch": 0.11059431524547804,
+      "grad_norm": 1.3746757507324219,
+      "learning_rate": 2.3453608247422683e-06,
+      "loss": 1.2494,
+      "step": 107
+    },
+    {
+      "epoch": 0.11162790697674418,
+      "grad_norm": 1.1640757322311401,
+      "learning_rate": 2.3711340206185566e-06,
+      "loss": 1.222,
+      "step": 108
+    },
+    {
+      "epoch": 0.11266149870801033,
+      "grad_norm": 1.5546257495880127,
+      "learning_rate": 2.3969072164948458e-06,
+      "loss": 1.2793,
+      "step": 109
+    },
+    {
+      "epoch": 0.11369509043927649,
+      "grad_norm": 1.4767199754714966,
+      "learning_rate": 2.422680412371134e-06,
+      "loss": 1.1982,
+      "step": 110
+    },
+    {
+      "epoch": 0.11472868217054263,
+      "grad_norm": 1.5099663734436035,
+      "learning_rate": 2.4484536082474228e-06,
+      "loss": 1.1992,
+      "step": 111
+    },
+    {
+      "epoch": 0.11576227390180878,
+      "grad_norm": 1.418818473815918,
+      "learning_rate": 2.4742268041237115e-06,
+      "loss": 1.2055,
+      "step": 112
+    },
+    {
+      "epoch": 0.11679586563307494,
+      "grad_norm": 1.59600830078125,
+      "learning_rate": 2.5e-06,
+      "loss": 1.2178,
+      "step": 113
+    },
+    {
+      "epoch": 0.11782945736434108,
+      "grad_norm": 1.8103954792022705,
+      "learning_rate": 2.525773195876289e-06,
+      "loss": 1.2114,
+      "step": 114
+    },
+    {
+      "epoch": 0.11886304909560723,
+      "grad_norm": 1.437061071395874,
+      "learning_rate": 2.5515463917525772e-06,
+      "loss": 1.2342,
+      "step": 115
+    },
+    {
+      "epoch": 0.11989664082687339,
+      "grad_norm": 1.6615889072418213,
+      "learning_rate": 2.577319587628866e-06,
+      "loss": 1.1291,
+      "step": 116
+    },
+    {
+      "epoch": 0.12093023255813953,
+      "grad_norm": 1.2814154624938965,
+      "learning_rate": 2.603092783505155e-06,
+      "loss": 1.1814,
+      "step": 117
+    },
+    {
+      "epoch": 0.12196382428940568,
+      "grad_norm": 1.5893689393997192,
+      "learning_rate": 2.628865979381444e-06,
+      "loss": 1.1617,
+      "step": 118
+    },
+    {
+      "epoch": 0.12299741602067184,
+      "grad_norm": 1.7495357990264893,
+      "learning_rate": 2.654639175257732e-06,
+      "loss": 1.1541,
+      "step": 119
+    },
+    {
+      "epoch": 0.12403100775193798,
+      "grad_norm": 1.8175143003463745,
+      "learning_rate": 2.680412371134021e-06,
+      "loss": 1.224,
+      "step": 120
+    },
+    {
+      "epoch": 0.12506459948320414,
+      "grad_norm": 1.4747029542922974,
+      "learning_rate": 2.7061855670103095e-06,
+      "loss": 1.2225,
+      "step": 121
+    },
+    {
+      "epoch": 0.12609819121447027,
+      "grad_norm": 1.5060474872589111,
+      "learning_rate": 2.731958762886598e-06,
+      "loss": 1.171,
+      "step": 122
+    },
+    {
+      "epoch": 0.12713178294573643,
+      "grad_norm": 1.3734132051467896,
+      "learning_rate": 2.757731958762887e-06,
+      "loss": 1.1808,
+      "step": 123
+    },
+    {
+      "epoch": 0.1281653746770026,
+      "grad_norm": 1.4248719215393066,
+      "learning_rate": 2.7835051546391757e-06,
+      "loss": 1.1876,
+      "step": 124
+    },
+    {
+      "epoch": 0.12919896640826872,
+      "grad_norm": 1.8982905149459839,
+      "learning_rate": 2.809278350515464e-06,
+      "loss": 1.1586,
+      "step": 125
+    },
+    {
+      "epoch": 0.13023255813953488,
+      "grad_norm": 1.4861031770706177,
+      "learning_rate": 2.8350515463917527e-06,
+      "loss": 1.1824,
+      "step": 126
+    },
+    {
+      "epoch": 0.13126614987080104,
+      "grad_norm": 1.9570235013961792,
+      "learning_rate": 2.8608247422680414e-06,
+      "loss": 1.1745,
+      "step": 127
+    },
+    {
+      "epoch": 0.13229974160206717,
+      "grad_norm": 1.6210845708847046,
+      "learning_rate": 2.8865979381443297e-06,
+      "loss": 1.1904,
+      "step": 128
+    },
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 1.4611804485321045,
+      "learning_rate": 2.912371134020619e-06,
+      "loss": 1.204,
+      "step": 129
+    },
+    {
+      "epoch": 0.1343669250645995,
+      "grad_norm": 1.3150776624679565,
+      "learning_rate": 2.9381443298969076e-06,
+      "loss": 1.2002,
+      "step": 130
+    },
+    {
+      "epoch": 0.13540051679586562,
+      "grad_norm": 1.3323931694030762,
+      "learning_rate": 2.9639175257731963e-06,
+      "loss": 1.1806,
+      "step": 131
+    },
+    {
+      "epoch": 0.13643410852713178,
+      "grad_norm": 1.5320898294448853,
+      "learning_rate": 2.9896907216494846e-06,
+      "loss": 1.1181,
+      "step": 132
+    },
+    {
+      "epoch": 0.13746770025839794,
+      "grad_norm": 1.431869626045227,
+      "learning_rate": 3.0154639175257733e-06,
+      "loss": 1.1735,
+      "step": 133
+    },
+    {
+      "epoch": 0.13850129198966407,
+      "grad_norm": 1.5655189752578735,
+      "learning_rate": 3.041237113402062e-06,
+      "loss": 1.1151,
+      "step": 134
+    },
+    {
+      "epoch": 0.13953488372093023,
+      "grad_norm": 1.8801727294921875,
+      "learning_rate": 3.067010309278351e-06,
+      "loss": 1.2014,
+      "step": 135
+    },
+    {
+      "epoch": 0.1405684754521964,
+      "grad_norm": 1.401580572128296,
+      "learning_rate": 3.0927835051546395e-06,
+      "loss": 1.1697,
+      "step": 136
+    },
+    {
+      "epoch": 0.14160206718346252,
+      "grad_norm": 1.3812285661697388,
+      "learning_rate": 3.118556701030928e-06,
+      "loss": 1.1642,
+      "step": 137
+    },
+    {
+      "epoch": 0.14263565891472868,
+      "grad_norm": 1.432713270187378,
+      "learning_rate": 3.1443298969072165e-06,
+      "loss": 1.1816,
+      "step": 138
+    },
+    {
+      "epoch": 0.14366925064599484,
+      "grad_norm": 1.2863351106643677,
+      "learning_rate": 3.170103092783505e-06,
+      "loss": 1.1337,
+      "step": 139
+    },
+    {
+      "epoch": 0.14470284237726097,
+      "grad_norm": 1.4150452613830566,
+      "learning_rate": 3.195876288659794e-06,
+      "loss": 1.1339,
+      "step": 140
+    },
+    {
+      "epoch": 0.14573643410852713,
+      "grad_norm": 1.4986978769302368,
+      "learning_rate": 3.221649484536083e-06,
+      "loss": 1.1486,
+      "step": 141
+    },
+    {
+      "epoch": 0.1467700258397933,
+      "grad_norm": 1.4928971529006958,
+      "learning_rate": 3.2474226804123714e-06,
+      "loss": 1.1505,
+      "step": 142
+    },
+    {
+      "epoch": 0.14780361757105942,
+      "grad_norm": 1.3250510692596436,
+      "learning_rate": 3.27319587628866e-06,
+      "loss": 1.1501,
+      "step": 143
+    },
+    {
+      "epoch": 0.14883720930232558,
+      "grad_norm": 1.3843717575073242,
+      "learning_rate": 3.298969072164949e-06,
+      "loss": 1.1425,
+      "step": 144
+    },
+    {
+      "epoch": 0.14987080103359174,
+      "grad_norm": 1.45151948928833,
+      "learning_rate": 3.324742268041237e-06,
+      "loss": 1.1412,
+      "step": 145
+    },
+    {
+      "epoch": 0.15090439276485787,
+      "grad_norm": 1.7031517028808594,
+      "learning_rate": 3.350515463917526e-06,
+      "loss": 1.1214,
+      "step": 146
+    },
+    {
+      "epoch": 0.15193798449612403,
+      "grad_norm": 1.735508918762207,
+      "learning_rate": 3.376288659793815e-06,
+      "loss": 1.1426,
+      "step": 147
+    },
+    {
+      "epoch": 0.1529715762273902,
+      "grad_norm": 1.4877556562423706,
+      "learning_rate": 3.4020618556701037e-06,
+      "loss": 1.1403,
+      "step": 148
+    },
+    {
+      "epoch": 0.15400516795865632,
+      "grad_norm": 1.4761852025985718,
+      "learning_rate": 3.427835051546392e-06,
+      "loss": 1.1189,
+      "step": 149
+    },
+    {
+      "epoch": 0.15503875968992248,
+      "grad_norm": 1.918549656867981,
+      "learning_rate": 3.4536082474226807e-06,
+      "loss": 1.177,
+      "step": 150
+    },
+    {
+      "epoch": 0.15503875968992248,
+      "eval_loss": 1.2169522047042847,
+      "eval_runtime": 52.6165,
+      "eval_samples_per_second": 19.005,
+      "eval_steps_per_second": 0.608,
+      "step": 150
+    },
+    {
+      "epoch": 0.15607235142118864,
+      "grad_norm": 1.6830838918685913,
+      "learning_rate": 3.4793814432989694e-06,
+      "loss": 1.1684,
+      "step": 151
+    },
+    {
+      "epoch": 0.15710594315245477,
+      "grad_norm": 1.520039677619934,
+      "learning_rate": 3.5051546391752577e-06,
+      "loss": 1.1557,
+      "step": 152
+    },
+    {
+      "epoch": 0.15813953488372093,
+      "grad_norm": 1.2740050554275513,
+      "learning_rate": 3.530927835051547e-06,
+      "loss": 1.1757,
+      "step": 153
+    },
+    {
+      "epoch": 0.1591731266149871,
+      "grad_norm": 1.6267930269241333,
+      "learning_rate": 3.5567010309278356e-06,
+      "loss": 1.1605,
+      "step": 154
+    },
+    {
+      "epoch": 0.16020671834625322,
+      "grad_norm": 1.392866611480713,
+      "learning_rate": 3.582474226804124e-06,
+      "loss": 1.1445,
+      "step": 155
+    },
+    {
+      "epoch": 0.16124031007751938,
+      "grad_norm": 1.5399266481399536,
+      "learning_rate": 3.6082474226804126e-06,
+      "loss": 1.1668,
+      "step": 156
+    },
+    {
+      "epoch": 0.16227390180878554,
+      "grad_norm": 1.5233243703842163,
+      "learning_rate": 3.6340206185567013e-06,
+      "loss": 1.1807,
+      "step": 157
+    },
+    {
+      "epoch": 0.16330749354005167,
+      "grad_norm": 1.4621553421020508,
+      "learning_rate": 3.6597938144329896e-06,
+      "loss": 1.1701,
+      "step": 158
+    },
+    {
+      "epoch": 0.16434108527131783,
+      "grad_norm": 1.4321812391281128,
+      "learning_rate": 3.6855670103092787e-06,
+      "loss": 1.1656,
+      "step": 159
+    },
+    {
+      "epoch": 0.165374677002584,
+      "grad_norm": 1.6212997436523438,
+      "learning_rate": 3.7113402061855674e-06,
+      "loss": 1.1508,
+      "step": 160
+    },
+    {
+      "epoch": 0.16640826873385012,
+      "grad_norm": 1.622518539428711,
+      "learning_rate": 3.737113402061856e-06,
+      "loss": 1.0932,
+      "step": 161
+    },
+    {
+      "epoch": 0.16744186046511628,
+      "grad_norm": 1.5313091278076172,
+      "learning_rate": 3.7628865979381445e-06,
+      "loss": 1.129,
+      "step": 162
+    },
+    {
+      "epoch": 0.16847545219638244,
+      "grad_norm": 1.4660190343856812,
+      "learning_rate": 3.788659793814433e-06,
+      "loss": 1.1718,
+      "step": 163
+    },
+    {
+      "epoch": 0.16950904392764857,
+      "grad_norm": 1.6606028079986572,
+      "learning_rate": 3.814432989690722e-06,
+      "loss": 1.141,
+      "step": 164
+    },
+    {
+      "epoch": 0.17054263565891473,
+      "grad_norm": 1.7094327211380005,
+      "learning_rate": 3.840206185567011e-06,
+      "loss": 1.0917,
+      "step": 165
+    },
+    {
+      "epoch": 0.17157622739018089,
+      "grad_norm": 1.4654054641723633,
+      "learning_rate": 3.865979381443299e-06,
+      "loss": 1.1037,
+      "step": 166
+    },
+    {
+      "epoch": 0.17260981912144702,
+      "grad_norm": 1.3869339227676392,
+      "learning_rate": 3.891752577319588e-06,
+      "loss": 1.1842,
+      "step": 167
+    },
+    {
+      "epoch": 0.17364341085271318,
+      "grad_norm": 1.8342238664627075,
+      "learning_rate": 3.917525773195877e-06,
+      "loss": 1.1069,
+      "step": 168
+    },
+    {
+      "epoch": 0.17467700258397933,
+      "grad_norm": 2.0523226261138916,
+      "learning_rate": 3.9432989690721655e-06,
+      "loss": 1.1172,
+      "step": 169
+    },
+    {
+      "epoch": 0.17571059431524547,
+      "grad_norm": 1.6128238439559937,
+      "learning_rate": 3.969072164948453e-06,
+      "loss": 1.0522,
+      "step": 170
+    },
+    {
+      "epoch": 0.17674418604651163,
+      "grad_norm": 1.5203567743301392,
+      "learning_rate": 3.994845360824743e-06,
+      "loss": 1.1296,
+      "step": 171
+    },
+    {
+      "epoch": 0.17777777777777778,
+      "grad_norm": 1.6611700057983398,
+      "learning_rate": 4.020618556701032e-06,
+      "loss": 1.1274,
+      "step": 172
+    },
+    {
+      "epoch": 0.17881136950904392,
+      "grad_norm": 1.6911481618881226,
+      "learning_rate": 4.04639175257732e-06,
+      "loss": 1.0755,
+      "step": 173
+    },
+    {
+      "epoch": 0.17984496124031008,
+      "grad_norm": 1.5666862726211548,
+      "learning_rate": 4.072164948453608e-06,
+      "loss": 1.1181,
+      "step": 174
+    },
+    {
+      "epoch": 0.18087855297157623,
+      "grad_norm": 1.5833418369293213,
+      "learning_rate": 4.097938144329897e-06,
+      "loss": 1.0879,
+      "step": 175
+    },
+    {
+      "epoch": 0.18191214470284237,
+      "grad_norm": 1.508384346961975,
+      "learning_rate": 4.123711340206186e-06,
+      "loss": 1.0751,
+      "step": 176
+    },
+    {
+      "epoch": 0.18294573643410852,
+      "grad_norm": 1.361509919166565,
+      "learning_rate": 4.149484536082475e-06,
+      "loss": 1.101,
+      "step": 177
+    },
+    {
+      "epoch": 0.18397932816537468,
+      "grad_norm": 1.4818835258483887,
+      "learning_rate": 4.175257731958763e-06,
+      "loss": 1.1135,
+      "step": 178
+    },
+    {
+      "epoch": 0.18501291989664082,
+      "grad_norm": 1.272712230682373,
+      "learning_rate": 4.201030927835052e-06,
+      "loss": 1.0984,
+      "step": 179
+    },
+    {
+      "epoch": 0.18604651162790697,
+      "grad_norm": 1.3766447305679321,
+      "learning_rate": 4.2268041237113405e-06,
+      "loss": 1.1007,
+      "step": 180
+    },
+    {
+      "epoch": 0.18708010335917313,
+      "grad_norm": 1.6823517084121704,
+      "learning_rate": 4.252577319587629e-06,
+      "loss": 1.1079,
+      "step": 181
+    },
+    {
+      "epoch": 0.18811369509043926,
+      "grad_norm": 1.7081917524337769,
+      "learning_rate": 4.278350515463918e-06,
+      "loss": 1.0972,
+      "step": 182
+    },
+    {
+      "epoch": 0.18914728682170542,
+      "grad_norm": 1.577996850013733,
+      "learning_rate": 4.304123711340207e-06,
+      "loss": 1.0842,
+      "step": 183
+    },
+    {
+      "epoch": 0.19018087855297158,
+      "grad_norm": 1.3267565965652466,
+      "learning_rate": 4.329896907216495e-06,
+      "loss": 1.1278,
+      "step": 184
+    },
+    {
+      "epoch": 0.19121447028423771,
+      "grad_norm": 1.3708866834640503,
+      "learning_rate": 4.355670103092784e-06,
+      "loss": 1.0795,
+      "step": 185
+    },
+    {
+      "epoch": 0.19224806201550387,
+      "grad_norm": 1.3076270818710327,
+      "learning_rate": 4.381443298969073e-06,
+      "loss": 1.1115,
+      "step": 186
+    },
+    {
+      "epoch": 0.19328165374677003,
+      "grad_norm": 1.4459667205810547,
+      "learning_rate": 4.407216494845361e-06,
+      "loss": 1.1004,
+      "step": 187
+    },
+    {
+      "epoch": 0.19431524547803616,
+      "grad_norm": 1.6593430042266846,
+      "learning_rate": 4.4329896907216494e-06,
+      "loss": 1.0942,
+      "step": 188
+    },
+    {
+      "epoch": 0.19534883720930232,
+      "grad_norm": 1.6031999588012695,
+      "learning_rate": 4.458762886597939e-06,
+      "loss": 1.0726,
+      "step": 189
+    },
+    {
+      "epoch": 0.19638242894056848,
+      "grad_norm": 1.75920569896698,
+      "learning_rate": 4.484536082474228e-06,
+      "loss": 1.0954,
+      "step": 190
+    },
+    {
+      "epoch": 0.1974160206718346,
+      "grad_norm": 1.6617308855056763,
+      "learning_rate": 4.510309278350516e-06,
+      "loss": 1.1088,
+      "step": 191
+    },
+    {
+      "epoch": 0.19844961240310077,
+      "grad_norm": 1.6344022750854492,
+      "learning_rate": 4.536082474226804e-06,
+      "loss": 1.102,
+      "step": 192
+    },
+    {
+      "epoch": 0.19948320413436693,
+      "grad_norm": 1.6076260805130005,
+      "learning_rate": 4.561855670103093e-06,
+      "loss": 1.1217,
+      "step": 193
+    },
+    {
+      "epoch": 0.20051679586563306,
+      "grad_norm": 1.5833053588867188,
+      "learning_rate": 4.587628865979382e-06,
+      "loss": 1.0837,
+      "step": 194
+    },
+    {
+      "epoch": 0.20155038759689922,
+      "grad_norm": 1.3770850896835327,
+      "learning_rate": 4.6134020618556705e-06,
+      "loss": 1.0462,
+      "step": 195
+    },
+    {
+      "epoch": 0.20258397932816538,
+      "grad_norm": 1.7390103340148926,
+      "learning_rate": 4.639175257731959e-06,
+      "loss": 1.0829,
+      "step": 196
+    },
+    {
+      "epoch": 0.2036175710594315,
+      "grad_norm": 1.3181084394454956,
+      "learning_rate": 4.664948453608248e-06,
+      "loss": 1.1128,
+      "step": 197
+    },
+    {
+      "epoch": 0.20465116279069767,
+      "grad_norm": 1.3936176300048828,
+      "learning_rate": 4.690721649484537e-06,
+      "loss": 1.1017,
+      "step": 198
+    },
+    {
+      "epoch": 0.20568475452196383,
+      "grad_norm": 1.4980775117874146,
+      "learning_rate": 4.716494845360825e-06,
+      "loss": 1.1231,
+      "step": 199
+    },
+    {
+      "epoch": 0.20671834625322996,
+      "grad_norm": 1.5333365201950073,
+      "learning_rate": 4.742268041237113e-06,
+      "loss": 1.0665,
+      "step": 200
+    },
+    {
+      "epoch": 0.20671834625322996,
+      "eval_loss": 1.1757628917694092,
+      "eval_runtime": 52.2634,
+      "eval_samples_per_second": 19.134,
+      "eval_steps_per_second": 0.612,
+      "step": 200
+    },
+    {
+      "epoch": 0.20775193798449612,
+      "grad_norm": 1.483267903327942,
+      "learning_rate": 4.768041237113403e-06,
+      "loss": 1.0646,
+      "step": 201
+    },
+    {
+      "epoch": 0.20878552971576228,
+      "grad_norm": 1.396013617515564,
+      "learning_rate": 4.7938144329896915e-06,
+      "loss": 1.0782,
+      "step": 202
+    },
+    {
+      "epoch": 0.2098191214470284,
+      "grad_norm": 1.5999109745025635,
+      "learning_rate": 4.81958762886598e-06,
+      "loss": 1.0694,
+      "step": 203
+    },
+    {
+      "epoch": 0.21085271317829457,
+      "grad_norm": 1.274971842765808,
+      "learning_rate": 4.845360824742268e-06,
+      "loss": 1.076,
+      "step": 204
+    },
+    {
+      "epoch": 0.21188630490956073,
+      "grad_norm": 1.6435425281524658,
+      "learning_rate": 4.871134020618557e-06,
+      "loss": 1.0432,
+      "step": 205
+    },
+    {
+      "epoch": 0.21291989664082686,
+      "grad_norm": 1.6180301904678345,
+      "learning_rate": 4.8969072164948455e-06,
+      "loss": 1.0912,
+      "step": 206
+    },
+    {
+      "epoch": 0.21395348837209302,
+      "grad_norm": 1.2818275690078735,
+      "learning_rate": 4.922680412371135e-06,
+      "loss": 1.0355,
+      "step": 207
+    },
+    {
+      "epoch": 0.21498708010335918,
+      "grad_norm": 1.4507101774215698,
+      "learning_rate": 4.948453608247423e-06,
+      "loss": 1.0734,
+      "step": 208
+    },
+    {
+      "epoch": 0.2160206718346253,
+      "grad_norm": 1.4751473665237427,
+      "learning_rate": 4.974226804123712e-06,
+      "loss": 1.0045,
+      "step": 209
+    },
+    {
+      "epoch": 0.21705426356589147,
+      "grad_norm": 1.3976224660873413,
+      "learning_rate": 5e-06,
+      "loss": 1.0913,
+      "step": 210
+    },
+    {
+      "epoch": 0.21808785529715763,
+      "grad_norm": 1.6017869710922241,
+      "learning_rate": 4.9971264367816096e-06,
+      "loss": 1.0493,
+      "step": 211
+    },
+    {
+      "epoch": 0.21912144702842376,
+      "grad_norm": 1.6266984939575195,
+      "learning_rate": 4.994252873563219e-06,
+      "loss": 1.0737,
+      "step": 212
+    },
+    {
+      "epoch": 0.22015503875968992,
+      "grad_norm": 1.6909797191619873,
+      "learning_rate": 4.991379310344828e-06,
+      "loss": 1.0571,
+      "step": 213
+    },
+    {
+      "epoch": 0.22118863049095608,
+      "grad_norm": 1.7753292322158813,
+      "learning_rate": 4.988505747126437e-06,
+      "loss": 1.0213,
+      "step": 214
+    },
+    {
+      "epoch": 0.2222222222222222,
+      "grad_norm": 1.4531023502349854,
+      "learning_rate": 4.985632183908046e-06,
+      "loss": 1.1229,
+      "step": 215
+    },
+    {
+      "epoch": 0.22325581395348837,
+      "grad_norm": 1.3566429615020752,
+      "learning_rate": 4.982758620689655e-06,
+      "loss": 1.0591,
+      "step": 216
+    },
+    {
+      "epoch": 0.22428940568475453,
+      "grad_norm": 1.3583022356033325,
+      "learning_rate": 4.9798850574712644e-06,
+      "loss": 1.0407,
+      "step": 217
+    },
+    {
+      "epoch": 0.22532299741602066,
+      "grad_norm": 1.7409709692001343,
+      "learning_rate": 4.977011494252874e-06,
+      "loss": 1.069,
+      "step": 218
+    },
+    {
+      "epoch": 0.22635658914728682,
+      "grad_norm": 1.4994089603424072,
+      "learning_rate": 4.9741379310344836e-06,
+      "loss": 1.1153,
+      "step": 219
+    },
+    {
+      "epoch": 0.22739018087855298,
+      "grad_norm": 1.6274763345718384,
+      "learning_rate": 4.971264367816092e-06,
+      "loss": 1.0447,
+      "step": 220
+    },
+    {
+      "epoch": 0.2284237726098191,
+      "grad_norm": 1.4050188064575195,
+      "learning_rate": 4.968390804597701e-06,
+      "loss": 1.0514,
+      "step": 221
+    },
+    {
+      "epoch": 0.22945736434108527,
+      "grad_norm": 1.9114052057266235,
+      "learning_rate": 4.965517241379311e-06,
+      "loss": 1.0414,
+      "step": 222
+    },
+    {
+      "epoch": 0.23049095607235143,
+      "grad_norm": 1.6414310932159424,
+      "learning_rate": 4.96264367816092e-06,
+      "loss": 1.1029,
+      "step": 223
+    },
+    {
+      "epoch": 0.23152454780361756,
+      "grad_norm": 1.844107985496521,
+      "learning_rate": 4.959770114942529e-06,
+      "loss": 1.0542,
+      "step": 224
+    },
+    {
+      "epoch": 0.23255813953488372,
+      "grad_norm": 1.539076566696167,
+      "learning_rate": 4.9568965517241384e-06,
+      "loss": 1.0478,
+      "step": 225
+    },
+    {
+      "epoch": 0.23359173126614988,
+      "grad_norm": 1.5214966535568237,
+      "learning_rate": 4.9540229885057476e-06,
+      "loss": 1.1052,
+      "step": 226
+    },
+    {
+      "epoch": 0.234625322997416,
+      "grad_norm": 1.9107270240783691,
+      "learning_rate": 4.951149425287357e-06,
+      "loss": 1.0664,
+      "step": 227
+    },
+    {
+      "epoch": 0.23565891472868217,
+      "grad_norm": 1.4986716508865356,
+      "learning_rate": 4.948275862068966e-06,
+      "loss": 1.0916,
+      "step": 228
+    },
+    {
+      "epoch": 0.23669250645994833,
+      "grad_norm": 1.4086743593215942,
+      "learning_rate": 4.945402298850575e-06,
+      "loss": 1.0774,
+      "step": 229
+    },
+    {
+      "epoch": 0.23772609819121446,
+      "grad_norm": 1.5430121421813965,
+      "learning_rate": 4.942528735632184e-06,
+      "loss": 1.0335,
+      "step": 230
+    },
+    {
+      "epoch": 0.23875968992248062,
+      "grad_norm": 1.9142231941223145,
+      "learning_rate": 4.939655172413793e-06,
+      "loss": 1.049,
+      "step": 231
+    },
+    {
+      "epoch": 0.23979328165374678,
+      "grad_norm": 1.5004005432128906,
+      "learning_rate": 4.936781609195403e-06,
+      "loss": 1.0741,
+      "step": 232
+    },
+    {
+      "epoch": 0.2408268733850129,
+      "grad_norm": 1.4257315397262573,
+      "learning_rate": 4.933908045977012e-06,
+      "loss": 0.9975,
+      "step": 233
+    },
+    {
+      "epoch": 0.24186046511627907,
+      "grad_norm": 1.5501189231872559,
+      "learning_rate": 4.931034482758621e-06,
+      "loss": 1.033,
+      "step": 234
+    },
+    {
+      "epoch": 0.24289405684754523,
+      "grad_norm": 1.5867544412612915,
+      "learning_rate": 4.92816091954023e-06,
+      "loss": 1.0641,
+      "step": 235
+    },
+    {
+      "epoch": 0.24392764857881136,
+      "grad_norm": 1.9389463663101196,
+      "learning_rate": 4.92528735632184e-06,
+      "loss": 0.9939,
+      "step": 236
+    },
+    {
+      "epoch": 0.24496124031007752,
+      "grad_norm": 1.5214310884475708,
+      "learning_rate": 4.922413793103449e-06,
+      "loss": 1.0662,
+      "step": 237
+    },
+    {
+      "epoch": 0.24599483204134368,
+      "grad_norm": 1.4389026165008545,
+      "learning_rate": 4.919540229885058e-06,
+      "loss": 1.044,
+      "step": 238
+    },
+    {
+      "epoch": 0.2470284237726098,
+      "grad_norm": 1.362501621246338,
+      "learning_rate": 4.9166666666666665e-06,
+      "loss": 1.0372,
+      "step": 239
+    },
+    {
+      "epoch": 0.24806201550387597,
+      "grad_norm": 1.5556910037994385,
+      "learning_rate": 4.9137931034482765e-06,
+      "loss": 1.0786,
+      "step": 240
+    },
+    {
+      "epoch": 0.24909560723514212,
+      "grad_norm": 1.384539246559143,
+      "learning_rate": 4.910919540229886e-06,
+      "loss": 1.06,
+      "step": 241
+    },
+    {
+      "epoch": 0.2501291989664083,
+      "grad_norm": 1.4042245149612427,
+      "learning_rate": 4.908045977011495e-06,
+      "loss": 1.0838,
+      "step": 242
+    },
+    {
+      "epoch": 0.25116279069767444,
+      "grad_norm": 1.3736196756362915,
+      "learning_rate": 4.905172413793104e-06,
+      "loss": 1.066,
+      "step": 243
+    },
+    {
+      "epoch": 0.25219638242894055,
+      "grad_norm": 1.4238249063491821,
+      "learning_rate": 4.902298850574713e-06,
+      "loss": 1.045,
+      "step": 244
+    },
+    {
+      "epoch": 0.2532299741602067,
+      "grad_norm": 1.3547841310501099,
+      "learning_rate": 4.899425287356322e-06,
+      "loss": 1.0253,
+      "step": 245
+    },
+    {
+      "epoch": 0.25426356589147286,
+      "grad_norm": 1.4367036819458008,
+      "learning_rate": 4.896551724137931e-06,
+      "loss": 1.0363,
+      "step": 246
+    },
+    {
+      "epoch": 0.255297157622739,
+      "grad_norm": 1.4458998441696167,
+      "learning_rate": 4.8936781609195405e-06,
+      "loss": 1.0069,
+      "step": 247
+    },
+    {
+      "epoch": 0.2563307493540052,
+      "grad_norm": 1.5451279878616333,
+      "learning_rate": 4.89080459770115e-06,
+      "loss": 1.0119,
+      "step": 248
+    },
+    {
+      "epoch": 0.25736434108527134,
+      "grad_norm": 1.3011225461959839,
+      "learning_rate": 4.887931034482759e-06,
+      "loss": 1.064,
+      "step": 249
+    },
+    {
+      "epoch": 0.25839793281653745,
+      "grad_norm": 1.2902860641479492,
+      "learning_rate": 4.885057471264369e-06,
+      "loss": 1.0082,
+      "step": 250
+    },
+    {
+      "epoch": 0.25839793281653745,
+      "eval_loss": 1.1425906419754028,
+      "eval_runtime": 52.1487,
+      "eval_samples_per_second": 19.176,
+      "eval_steps_per_second": 0.614,
+      "step": 250
+    },
+    {
+      "epoch": 0.2594315245478036,
+      "grad_norm": 1.3218801021575928,
+      "learning_rate": 4.882183908045978e-06,
+      "loss": 1.0107,
+      "step": 251
+    },
+    {
+      "epoch": 0.26046511627906976,
+      "grad_norm": 1.5538569688796997,
+      "learning_rate": 4.879310344827586e-06,
+      "loss": 1.0203,
+      "step": 252
+    },
+    {
+      "epoch": 0.2614987080103359,
+      "grad_norm": 1.4206135272979736,
+      "learning_rate": 4.876436781609195e-06,
+      "loss": 1.1056,
+      "step": 253
+    },
+    {
+      "epoch": 0.2625322997416021,
+      "grad_norm": 1.5996060371398926,
+      "learning_rate": 4.873563218390805e-06,
+      "loss": 1.0236,
+      "step": 254
+    },
+    {
+      "epoch": 0.26356589147286824,
+      "grad_norm": 1.5735429525375366,
+      "learning_rate": 4.8706896551724145e-06,
+      "loss": 1.0763,
+      "step": 255
+    },
+    {
+      "epoch": 0.26459948320413434,
+      "grad_norm": 1.3383245468139648,
+      "learning_rate": 4.867816091954024e-06,
+      "loss": 1.0463,
+      "step": 256
+    },
+    {
+      "epoch": 0.2656330749354005,
+      "grad_norm": 1.752362847328186,
+      "learning_rate": 4.864942528735633e-06,
+      "loss": 1.0451,
+      "step": 257
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.4010084867477417,
+      "learning_rate": 4.862068965517242e-06,
+      "loss": 1.043,
+      "step": 258
+    },
+    {
+      "epoch": 0.2677002583979328,
+      "grad_norm": 1.4108176231384277,
+      "learning_rate": 4.859195402298851e-06,
+      "loss": 1.0534,
+      "step": 259
+    },
+    {
+      "epoch": 0.268733850129199,
+      "grad_norm": 1.5272636413574219,
+      "learning_rate": 4.85632183908046e-06,
+      "loss": 1.0264,
+      "step": 260
+    },
+    {
+      "epoch": 0.26976744186046514,
+      "grad_norm": 1.2899655103683472,
+      "learning_rate": 4.853448275862069e-06,
+      "loss": 1.0403,
+      "step": 261
+    },
+    {
+      "epoch": 0.27080103359173124,
+      "grad_norm": 1.5390360355377197,
+      "learning_rate": 4.8505747126436785e-06,
+      "loss": 1.0281,
+      "step": 262
+    },
+    {
+      "epoch": 0.2718346253229974,
+      "grad_norm": 1.4357856512069702,
+      "learning_rate": 4.847701149425288e-06,
+      "loss": 1.0423,
+      "step": 263
+    },
+    {
+      "epoch": 0.27286821705426356,
+      "grad_norm": 1.2563819885253906,
+      "learning_rate": 4.844827586206897e-06,
+      "loss": 1.0487,
+      "step": 264
+    },
+    {
+      "epoch": 0.2739018087855297,
+      "grad_norm": 1.605078935623169,
+      "learning_rate": 4.841954022988506e-06,
+      "loss": 1.0626,
+      "step": 265
+    },
+    {
+      "epoch": 0.2749354005167959,
+      "grad_norm": 1.4388753175735474,
+      "learning_rate": 4.839080459770115e-06,
+      "loss": 1.0494,
+      "step": 266
+    },
+    {
+      "epoch": 0.27596899224806204,
+      "grad_norm": 1.735318660736084,
+      "learning_rate": 4.836206896551724e-06,
+      "loss": 1.0359,
+      "step": 267
+    },
+    {
+      "epoch": 0.27700258397932814,
+      "grad_norm": 1.7299683094024658,
+      "learning_rate": 4.833333333333333e-06,
+      "loss": 1.005,
+      "step": 268
+    },
+    {
+      "epoch": 0.2780361757105943,
+      "grad_norm": 1.3382335901260376,
+      "learning_rate": 4.830459770114943e-06,
+      "loss": 1.0523,
+      "step": 269
+    },
+    {
+      "epoch": 0.27906976744186046,
+      "grad_norm": 1.3449649810791016,
+      "learning_rate": 4.8275862068965525e-06,
+      "loss": 1.042,
+      "step": 270
+    },
+    {
+      "epoch": 0.2801033591731266,
+      "grad_norm": 1.615478277206421,
+      "learning_rate": 4.824712643678161e-06,
+      "loss": 1.042,
+      "step": 271
+    },
+    {
+      "epoch": 0.2811369509043928,
+      "grad_norm": 1.556579351425171,
+      "learning_rate": 4.82183908045977e-06,
+      "loss": 0.9972,
+      "step": 272
+    },
+    {
+      "epoch": 0.28217054263565894,
+      "grad_norm": 1.4196879863739014,
+      "learning_rate": 4.81896551724138e-06,
+      "loss": 1.0311,
+      "step": 273
+    },
+    {
+      "epoch": 0.28320413436692504,
+      "grad_norm": 1.2686911821365356,
+      "learning_rate": 4.816091954022989e-06,
+      "loss": 1.0503,
+      "step": 274
+    },
+    {
+      "epoch": 0.2842377260981912,
+      "grad_norm": 1.1563490629196167,
+      "learning_rate": 4.813218390804598e-06,
+      "loss": 1.0183,
+      "step": 275
+    },
+    {
+      "epoch": 0.28527131782945736,
+      "grad_norm": 1.4043651819229126,
+      "learning_rate": 4.810344827586207e-06,
+      "loss": 1.0437,
+      "step": 276
+    },
+    {
+      "epoch": 0.2863049095607235,
+      "grad_norm": 1.4565726518630981,
+      "learning_rate": 4.8074712643678165e-06,
+      "loss": 1.0212,
+      "step": 277
+    },
+    {
+      "epoch": 0.2873385012919897,
+      "grad_norm": 1.4200972318649292,
+      "learning_rate": 4.804597701149426e-06,
+      "loss": 1.0355,
+      "step": 278
+    },
+    {
+      "epoch": 0.28837209302325584,
+      "grad_norm": 1.403757095336914,
+      "learning_rate": 4.801724137931035e-06,
+      "loss": 1.0271,
+      "step": 279
+    },
+    {
+      "epoch": 0.28940568475452194,
+      "grad_norm": 1.2572818994522095,
+      "learning_rate": 4.798850574712644e-06,
+      "loss": 1.0319,
+      "step": 280
+    },
+    {
+      "epoch": 0.2904392764857881,
+      "grad_norm": 1.3813985586166382,
+      "learning_rate": 4.795977011494253e-06,
+      "loss": 1.0502,
+      "step": 281
+    },
+    {
+      "epoch": 0.29147286821705426,
+      "grad_norm": 1.8169054985046387,
+      "learning_rate": 4.793103448275862e-06,
+      "loss": 1.0395,
+      "step": 282
+    },
+    {
+      "epoch": 0.2925064599483204,
+      "grad_norm": 1.2015506029129028,
+      "learning_rate": 4.790229885057472e-06,
+      "loss": 1.0498,
+      "step": 283
+    },
+    {
+      "epoch": 0.2935400516795866,
+      "grad_norm": 1.4416213035583496,
+      "learning_rate": 4.7873563218390805e-06,
+      "loss": 0.9921,
+      "step": 284
+    },
+    {
+      "epoch": 0.29457364341085274,
+      "grad_norm": 1.3626724481582642,
+      "learning_rate": 4.78448275862069e-06,
+      "loss": 1.0185,
+      "step": 285
+    },
+    {
+      "epoch": 0.29560723514211884,
+      "grad_norm": 1.3624345064163208,
+      "learning_rate": 4.781609195402299e-06,
+      "loss": 1.0376,
+      "step": 286
+    },
+    {
+      "epoch": 0.296640826873385,
+      "grad_norm": 1.5711549520492554,
+      "learning_rate": 4.778735632183909e-06,
+      "loss": 1.0078,
+      "step": 287
+    },
+    {
+      "epoch": 0.29767441860465116,
+      "grad_norm": 1.721814513206482,
+      "learning_rate": 4.775862068965518e-06,
+      "loss": 1.0828,
+      "step": 288
+    },
+    {
+      "epoch": 0.2987080103359173,
+      "grad_norm": 1.5979557037353516,
+      "learning_rate": 4.772988505747127e-06,
+      "loss": 1.0135,
+      "step": 289
+    },
+    {
+      "epoch": 0.2997416020671835,
+      "grad_norm": 1.3976504802703857,
+      "learning_rate": 4.770114942528735e-06,
+      "loss": 1.0213,
+      "step": 290
+    },
+    {
+      "epoch": 0.30077519379844964,
+      "grad_norm": 1.361649513244629,
+      "learning_rate": 4.767241379310345e-06,
+      "loss": 1.0308,
+      "step": 291
+    },
+    {
+      "epoch": 0.30180878552971574,
+      "grad_norm": 1.3206751346588135,
+      "learning_rate": 4.7643678160919545e-06,
+      "loss": 0.9987,
+      "step": 292
+    },
+    {
+      "epoch": 0.3028423772609819,
+      "grad_norm": 1.720141053199768,
+      "learning_rate": 4.761494252873564e-06,
+      "loss": 1.0413,
+      "step": 293
+    },
+    {
+      "epoch": 0.30387596899224806,
+      "grad_norm": 1.5092377662658691,
+      "learning_rate": 4.758620689655173e-06,
+      "loss": 0.9938,
+      "step": 294
+    },
+    {
+      "epoch": 0.3049095607235142,
+      "grad_norm": 1.4403694868087769,
+      "learning_rate": 4.755747126436782e-06,
+      "loss": 1.0379,
+      "step": 295
+    },
+    {
+      "epoch": 0.3059431524547804,
+      "grad_norm": 1.3464564085006714,
+      "learning_rate": 4.752873563218391e-06,
+      "loss": 1.0645,
+      "step": 296
+    },
+    {
+      "epoch": 0.30697674418604654,
+      "grad_norm": 1.3892803192138672,
+      "learning_rate": 4.75e-06,
+      "loss": 0.9331,
+      "step": 297
+    },
+    {
+      "epoch": 0.30801033591731264,
+      "grad_norm": 1.650929570198059,
+      "learning_rate": 4.747126436781609e-06,
+      "loss": 1.0249,
+      "step": 298
+    },
+    {
+      "epoch": 0.3090439276485788,
+      "grad_norm": 1.3314130306243896,
+      "learning_rate": 4.7442528735632186e-06,
+      "loss": 1.0332,
+      "step": 299
+    },
+    {
+      "epoch": 0.31007751937984496,
+      "grad_norm": 1.6108005046844482,
+      "learning_rate": 4.741379310344828e-06,
+      "loss": 1.0138,
+      "step": 300
+    },
+    {
+      "epoch": 0.31007751937984496,
+      "eval_loss": 1.1177282333374023,
+      "eval_runtime": 52.0926,
+      "eval_samples_per_second": 19.197,
+      "eval_steps_per_second": 0.614,
+      "step": 300
+    },
+    {
+      "epoch": 0.3111111111111111,
+      "grad_norm": 1.2376863956451416,
+      "learning_rate": 4.738505747126438e-06,
+      "loss": 1.0149,
+      "step": 301
+    },
+    {
+      "epoch": 0.3121447028423773,
+      "grad_norm": 1.3628637790679932,
+      "learning_rate": 4.735632183908047e-06,
+      "loss": 1.0157,
+      "step": 302
+    },
+    {
+      "epoch": 0.31317829457364343,
+      "grad_norm": 1.2822990417480469,
+      "learning_rate": 4.732758620689655e-06,
+      "loss": 1.0286,
+      "step": 303
+    },
+    {
+      "epoch": 0.31421188630490954,
+      "grad_norm": 1.2702423334121704,
+      "learning_rate": 4.729885057471264e-06,
+      "loss": 1.0365,
+      "step": 304
+    },
+    {
+      "epoch": 0.3152454780361757,
+      "grad_norm": 1.2091736793518066,
+      "learning_rate": 4.727011494252874e-06,
+      "loss": 0.995,
+      "step": 305
+    },
+    {
+      "epoch": 0.31627906976744186,
+      "grad_norm": 1.223698377609253,
+      "learning_rate": 4.724137931034483e-06,
+      "loss": 1.016,
+      "step": 306
+    },
+    {
+      "epoch": 0.317312661498708,
+      "grad_norm": 1.7964216470718384,
+      "learning_rate": 4.7212643678160926e-06,
+      "loss": 1.0564,
+      "step": 307
+    },
+    {
+      "epoch": 0.3183462532299742,
+      "grad_norm": 1.693984866142273,
+      "learning_rate": 4.718390804597702e-06,
+      "loss": 1.0381,
+      "step": 308
+    },
+    {
+      "epoch": 0.31937984496124033,
+      "grad_norm": 1.3786847591400146,
+      "learning_rate": 4.715517241379311e-06,
+      "loss": 1.0385,
+      "step": 309
+    },
+    {
+      "epoch": 0.32041343669250644,
+      "grad_norm": 1.5243544578552246,
+      "learning_rate": 4.71264367816092e-06,
+      "loss": 1.0313,
+      "step": 310
+    },
+    {
+      "epoch": 0.3214470284237726,
+      "grad_norm": 1.2947908639907837,
+      "learning_rate": 4.709770114942529e-06,
+      "loss": 1.0126,
+      "step": 311
+    },
+    {
+      "epoch": 0.32248062015503876,
+      "grad_norm": 1.5751510858535767,
+      "learning_rate": 4.706896551724138e-06,
+      "loss": 1.0268,
+      "step": 312
+    },
+    {
+      "epoch": 0.3235142118863049,
+      "grad_norm": 1.335180401802063,
+      "learning_rate": 4.7040229885057474e-06,
+      "loss": 1.0474,
+      "step": 313
+    },
+    {
+      "epoch": 0.3245478036175711,
+      "grad_norm": 1.4872468709945679,
+      "learning_rate": 4.7011494252873566e-06,
+      "loss": 1.0709,
+      "step": 314
+    },
+    {
+      "epoch": 0.32558139534883723,
+      "grad_norm": 1.5406771898269653,
+      "learning_rate": 4.698275862068966e-06,
+      "loss": 1.032,
+      "step": 315
+    },
+    {
+      "epoch": 0.32661498708010334,
+      "grad_norm": 1.2771215438842773,
+      "learning_rate": 4.695402298850575e-06,
+      "loss": 1.0122,
+      "step": 316
+    },
+    {
+      "epoch": 0.3276485788113695,
+      "grad_norm": 1.4511051177978516,
+      "learning_rate": 4.692528735632184e-06,
+      "loss": 1.0457,
+      "step": 317
+    },
+    {
+      "epoch": 0.32868217054263565,
+      "grad_norm": 1.2866500616073608,
+      "learning_rate": 4.689655172413793e-06,
+      "loss": 1.0154,
+      "step": 318
+    },
+    {
+      "epoch": 0.3297157622739018,
+      "grad_norm": 1.3319385051727295,
+      "learning_rate": 4.686781609195402e-06,
+      "loss": 1.0351,
+      "step": 319
+    },
+    {
+      "epoch": 0.330749354005168,
+      "grad_norm": 1.499594807624817,
+      "learning_rate": 4.683908045977012e-06,
+      "loss": 0.9905,
+      "step": 320
+    },
+    {
+      "epoch": 0.33178294573643413,
+      "grad_norm": 1.4195674657821655,
+      "learning_rate": 4.6810344827586214e-06,
+      "loss": 0.9723,
+      "step": 321
+    },
+    {
+      "epoch": 0.33281653746770024,
+      "grad_norm": 1.398489236831665,
+      "learning_rate": 4.67816091954023e-06,
+      "loss": 0.9869,
+      "step": 322
+    },
+    {
+      "epoch": 0.3338501291989664,
+      "grad_norm": 1.5325692892074585,
+      "learning_rate": 4.675287356321839e-06,
+      "loss": 0.9852,
+      "step": 323
+    },
+    {
+      "epoch": 0.33488372093023255,
+      "grad_norm": 1.5350309610366821,
+      "learning_rate": 4.672413793103449e-06,
+      "loss": 1.044,
+      "step": 324
+    },
+    {
+      "epoch": 0.3359173126614987,
+      "grad_norm": 1.534777045249939,
+      "learning_rate": 4.669540229885058e-06,
+      "loss": 0.9982,
+      "step": 325
+    },
+    {
+      "epoch": 0.33695090439276487,
+      "grad_norm": 1.7731380462646484,
+      "learning_rate": 4.666666666666667e-06,
+      "loss": 1.0407,
+      "step": 326
+    },
+    {
+      "epoch": 0.33798449612403103,
+      "grad_norm": 1.6210086345672607,
+      "learning_rate": 4.663793103448276e-06,
+      "loss": 0.9839,
+      "step": 327
+    },
+    {
+      "epoch": 0.33901808785529713,
+      "grad_norm": 1.139635682106018,
+      "learning_rate": 4.6609195402298855e-06,
+      "loss": 0.9218,
+      "step": 328
+    },
+    {
+      "epoch": 0.3400516795865633,
+      "grad_norm": 1.3825052976608276,
+      "learning_rate": 4.658045977011495e-06,
+      "loss": 0.9853,
+      "step": 329
+    },
+    {
+      "epoch": 0.34108527131782945,
+      "grad_norm": 1.6630350351333618,
+      "learning_rate": 4.655172413793104e-06,
+      "loss": 1.0007,
+      "step": 330
+    },
+    {
+      "epoch": 0.3421188630490956,
+      "grad_norm": 1.585688591003418,
+      "learning_rate": 4.652298850574713e-06,
+      "loss": 1.0594,
+      "step": 331
+    },
+    {
+      "epoch": 0.34315245478036177,
+      "grad_norm": 1.8591464757919312,
+      "learning_rate": 4.649425287356322e-06,
+      "loss": 1.0056,
+      "step": 332
+    },
+    {
+      "epoch": 0.34418604651162793,
+      "grad_norm": 1.5832587480545044,
+      "learning_rate": 4.646551724137931e-06,
+      "loss": 0.9986,
+      "step": 333
+    },
+    {
+      "epoch": 0.34521963824289403,
+      "grad_norm": 1.7467713356018066,
+      "learning_rate": 4.643678160919541e-06,
+      "loss": 0.9925,
+      "step": 334
+    },
+    {
+      "epoch": 0.3462532299741602,
+      "grad_norm": 1.3366422653198242,
+      "learning_rate": 4.6408045977011495e-06,
+      "loss": 1.0477,
+      "step": 335
+    },
+    {
+      "epoch": 0.34728682170542635,
+      "grad_norm": 1.473027229309082,
+      "learning_rate": 4.637931034482759e-06,
+      "loss": 0.9695,
+      "step": 336
+    },
+    {
+      "epoch": 0.3483204134366925,
+      "grad_norm": 1.5177369117736816,
+      "learning_rate": 4.635057471264368e-06,
+      "loss": 1.0238,
+      "step": 337
+    },
+    {
+      "epoch": 0.34935400516795867,
+      "grad_norm": 1.9143269062042236,
+      "learning_rate": 4.632183908045978e-06,
+      "loss": 1.0595,
+      "step": 338
+    },
+    {
+      "epoch": 0.35038759689922483,
+      "grad_norm": 1.4199109077453613,
+      "learning_rate": 4.629310344827587e-06,
+      "loss": 1.02,
+      "step": 339
+    },
+    {
+      "epoch": 0.35142118863049093,
+      "grad_norm": 1.3345683813095093,
+      "learning_rate": 4.626436781609196e-06,
+      "loss": 0.9948,
+      "step": 340
+    },
+    {
+      "epoch": 0.3524547803617571,
+      "grad_norm": 1.3201162815093994,
+      "learning_rate": 4.623563218390805e-06,
+      "loss": 0.9978,
+      "step": 341
+    },
+    {
+      "epoch": 0.35348837209302325,
+      "grad_norm": 1.7195253372192383,
+      "learning_rate": 4.620689655172414e-06,
+      "loss": 0.9977,
+      "step": 342
+    },
+    {
+      "epoch": 0.3545219638242894,
+      "grad_norm": 1.3972502946853638,
+      "learning_rate": 4.6178160919540235e-06,
+      "loss": 1.0299,
+      "step": 343
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "grad_norm": 1.4695457220077515,
+      "learning_rate": 4.614942528735633e-06,
+      "loss": 1.0164,
+      "step": 344
+    },
+    {
+      "epoch": 0.35658914728682173,
+      "grad_norm": 1.5302752256393433,
+      "learning_rate": 4.612068965517242e-06,
+      "loss": 1.0209,
+      "step": 345
+    },
+    {
+      "epoch": 0.35762273901808783,
+      "grad_norm": 1.2683886289596558,
+      "learning_rate": 4.609195402298851e-06,
+      "loss": 0.9828,
+      "step": 346
+    },
+    {
+      "epoch": 0.358656330749354,
+      "grad_norm": 1.2424372434616089,
+      "learning_rate": 4.60632183908046e-06,
+      "loss": 0.9937,
+      "step": 347
+    },
+    {
+      "epoch": 0.35968992248062015,
+      "grad_norm": 1.2774847745895386,
+      "learning_rate": 4.603448275862069e-06,
+      "loss": 1.0385,
+      "step": 348
+    },
+    {
+      "epoch": 0.3607235142118863,
+      "grad_norm": 1.1979737281799316,
+      "learning_rate": 4.600574712643678e-06,
+      "loss": 0.9658,
+      "step": 349
+    },
+    {
+      "epoch": 0.36175710594315247,
+      "grad_norm": 1.6517276763916016,
+      "learning_rate": 4.5977011494252875e-06,
+      "loss": 0.9967,
+      "step": 350
+    },
+    {
+      "epoch": 0.36175710594315247,
+      "eval_loss": 1.1045842170715332,
+      "eval_runtime": 52.4997,
+      "eval_samples_per_second": 19.048,
+      "eval_steps_per_second": 0.61,
+      "step": 350
+    },
+    {
+      "epoch": 0.3627906976744186,
+      "grad_norm": 1.2296693325042725,
+      "learning_rate": 4.594827586206897e-06,
+      "loss": 0.9675,
+      "step": 351
+    },
+    {
+      "epoch": 0.36382428940568473,
+      "grad_norm": 1.367437720298767,
+      "learning_rate": 4.591954022988507e-06,
+      "loss": 1.0251,
+      "step": 352
+    },
+    {
+      "epoch": 0.3648578811369509,
+      "grad_norm": 1.3952935934066772,
+      "learning_rate": 4.589080459770116e-06,
+      "loss": 1.0408,
+      "step": 353
+    },
+    {
+      "epoch": 0.36589147286821705,
+      "grad_norm": 1.287678837776184,
+      "learning_rate": 4.586206896551724e-06,
+      "loss": 0.9839,
+      "step": 354
+    },
+    {
+      "epoch": 0.3669250645994832,
+      "grad_norm": 1.599214792251587,
+      "learning_rate": 4.583333333333333e-06,
+      "loss": 1.0125,
+      "step": 355
+    },
+    {
+      "epoch": 0.36795865633074937,
+      "grad_norm": 1.6125950813293457,
+      "learning_rate": 4.580459770114943e-06,
+      "loss": 1.0108,
+      "step": 356
+    },
+    {
+      "epoch": 0.3689922480620155,
+      "grad_norm": 1.167265772819519,
+      "learning_rate": 4.577586206896552e-06,
+      "loss": 0.9809,
+      "step": 357
+    },
+    {
+      "epoch": 0.37002583979328163,
+      "grad_norm": 1.3578706979751587,
+      "learning_rate": 4.5747126436781615e-06,
+      "loss": 1.0054,
+      "step": 358
+    },
+    {
+      "epoch": 0.3710594315245478,
+      "grad_norm": 1.5812793970108032,
+      "learning_rate": 4.571839080459771e-06,
+      "loss": 0.9962,
+      "step": 359
+    },
+    {
+      "epoch": 0.37209302325581395,
+      "grad_norm": 1.6925969123840332,
+      "learning_rate": 4.56896551724138e-06,
+      "loss": 1.0039,
+      "step": 360
+    },
+    {
+      "epoch": 0.3731266149870801,
+      "grad_norm": 1.5617161989212036,
+      "learning_rate": 4.566091954022989e-06,
+      "loss": 1.0123,
+      "step": 361
+    },
+    {
+      "epoch": 0.37416020671834627,
+      "grad_norm": 1.2854423522949219,
+      "learning_rate": 4.563218390804598e-06,
+      "loss": 1.0074,
+      "step": 362
+    },
+    {
+      "epoch": 0.3751937984496124,
+      "grad_norm": 1.5997135639190674,
+      "learning_rate": 4.560344827586207e-06,
+      "loss": 0.9894,
+      "step": 363
+    },
+    {
+      "epoch": 0.37622739018087853,
+      "grad_norm": 1.5839483737945557,
+      "learning_rate": 4.557471264367816e-06,
+      "loss": 1.0093,
+      "step": 364
+    },
+    {
+      "epoch": 0.3772609819121447,
+      "grad_norm": 1.3355575799942017,
+      "learning_rate": 4.5545977011494255e-06,
+      "loss": 1.0131,
+      "step": 365
+    },
+    {
+      "epoch": 0.37829457364341085,
+      "grad_norm": 1.3378883600234985,
+      "learning_rate": 4.551724137931035e-06,
+      "loss": 1.0251,
+      "step": 366
+    },
+    {
+      "epoch": 0.379328165374677,
+      "grad_norm": 1.5703293085098267,
+      "learning_rate": 4.548850574712644e-06,
+      "loss": 1.0052,
+      "step": 367
+    },
+    {
+      "epoch": 0.38036175710594317,
+      "grad_norm": 1.6331372261047363,
+      "learning_rate": 4.545977011494253e-06,
+      "loss": 1.0063,
+      "step": 368
+    },
+    {
+      "epoch": 0.3813953488372093,
+      "grad_norm": 1.4740118980407715,
+      "learning_rate": 4.543103448275862e-06,
+      "loss": 1.0024,
+      "step": 369
+    },
+    {
+      "epoch": 0.38242894056847543,
+      "grad_norm": 1.3222692012786865,
+      "learning_rate": 4.540229885057471e-06,
+      "loss": 0.982,
+      "step": 370
+    },
+    {
+      "epoch": 0.3834625322997416,
+      "grad_norm": 1.4858118295669556,
+      "learning_rate": 4.537356321839081e-06,
+      "loss": 1.033,
+      "step": 371
+    },
+    {
+      "epoch": 0.38449612403100775,
+      "grad_norm": 1.201407551765442,
+      "learning_rate": 4.53448275862069e-06,
+      "loss": 0.9925,
+      "step": 372
+    },
+    {
+      "epoch": 0.3855297157622739,
+      "grad_norm": 1.276807427406311,
+      "learning_rate": 4.5316091954022995e-06,
+      "loss": 1.0374,
+      "step": 373
+    },
+    {
+      "epoch": 0.38656330749354006,
+      "grad_norm": 1.306963324546814,
+      "learning_rate": 4.528735632183908e-06,
+      "loss": 0.9913,
+      "step": 374
+    },
+    {
+      "epoch": 0.3875968992248062,
+      "grad_norm": 1.4419054985046387,
+      "learning_rate": 4.525862068965518e-06,
+      "loss": 1.0021,
+      "step": 375
+    },
+    {
+      "epoch": 0.3886304909560723,
+      "grad_norm": 1.1594988107681274,
+      "learning_rate": 4.522988505747127e-06,
+      "loss": 0.967,
+      "step": 376
+    },
+    {
+      "epoch": 0.3896640826873385,
+      "grad_norm": 1.2666174173355103,
+      "learning_rate": 4.520114942528736e-06,
+      "loss": 1.0206,
+      "step": 377
+    },
+    {
+      "epoch": 0.39069767441860465,
+      "grad_norm": 1.4637069702148438,
+      "learning_rate": 4.517241379310345e-06,
+      "loss": 1.0182,
+      "step": 378
+    },
+    {
+      "epoch": 0.3917312661498708,
+      "grad_norm": 1.3480263948440552,
+      "learning_rate": 4.514367816091954e-06,
+      "loss": 0.9642,
+      "step": 379
+    },
+    {
+      "epoch": 0.39276485788113696,
+      "grad_norm": 1.3101832866668701,
+      "learning_rate": 4.5114942528735635e-06,
+      "loss": 0.9631,
+      "step": 380
+    },
+    {
+      "epoch": 0.3937984496124031,
+      "grad_norm": 1.2533233165740967,
+      "learning_rate": 4.508620689655173e-06,
+      "loss": 0.9842,
+      "step": 381
+    },
+    {
+      "epoch": 0.3948320413436692,
+      "grad_norm": 1.2661020755767822,
+      "learning_rate": 4.505747126436782e-06,
+      "loss": 0.9842,
+      "step": 382
+    },
+    {
+      "epoch": 0.3958656330749354,
+      "grad_norm": 1.4904688596725464,
+      "learning_rate": 4.502873563218391e-06,
+      "loss": 0.9785,
+      "step": 383
+    },
+    {
+      "epoch": 0.39689922480620154,
+      "grad_norm": 1.2895351648330688,
+      "learning_rate": 4.5e-06,
+      "loss": 0.9755,
+      "step": 384
+    },
+    {
+      "epoch": 0.3979328165374677,
+      "grad_norm": 1.186653971672058,
+      "learning_rate": 4.49712643678161e-06,
+      "loss": 0.9699,
+      "step": 385
+    },
+    {
+      "epoch": 0.39896640826873386,
+      "grad_norm": 1.6472855806350708,
+      "learning_rate": 4.494252873563218e-06,
+      "loss": 0.9963,
+      "step": 386
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.535745620727539,
+      "learning_rate": 4.4913793103448275e-06,
+      "loss": 0.9925,
+      "step": 387
+    },
+    {
+      "epoch": 0.4010335917312661,
+      "grad_norm": 1.3963381052017212,
+      "learning_rate": 4.488505747126437e-06,
+      "loss": 0.9926,
+      "step": 388
+    },
+    {
+      "epoch": 0.4020671834625323,
+      "grad_norm": 1.2652450799942017,
+      "learning_rate": 4.485632183908047e-06,
+      "loss": 1.0224,
+      "step": 389
+    },
+    {
+      "epoch": 0.40310077519379844,
+      "grad_norm": 1.329743504524231,
+      "learning_rate": 4.482758620689656e-06,
+      "loss": 0.9473,
+      "step": 390
+    },
+    {
+      "epoch": 0.4041343669250646,
+      "grad_norm": 1.471661925315857,
+      "learning_rate": 4.479885057471265e-06,
+      "loss": 0.9788,
+      "step": 391
+    },
+    {
+      "epoch": 0.40516795865633076,
+      "grad_norm": 1.3997739553451538,
+      "learning_rate": 4.477011494252874e-06,
+      "loss": 0.9918,
+      "step": 392
+    },
+    {
+      "epoch": 0.4062015503875969,
+      "grad_norm": 1.3732025623321533,
+      "learning_rate": 4.474137931034483e-06,
+      "loss": 0.9972,
+      "step": 393
+    },
+    {
+      "epoch": 0.407235142118863,
+      "grad_norm": 1.3872554302215576,
+      "learning_rate": 4.471264367816092e-06,
+      "loss": 0.9695,
+      "step": 394
+    },
+    {
+      "epoch": 0.4082687338501292,
+      "grad_norm": 1.3868608474731445,
+      "learning_rate": 4.4683908045977016e-06,
+      "loss": 1.0138,
+      "step": 395
+    },
+    {
+      "epoch": 0.40930232558139534,
+      "grad_norm": 1.59169602394104,
+      "learning_rate": 4.465517241379311e-06,
+      "loss": 1.0182,
+      "step": 396
+    },
+    {
+      "epoch": 0.4103359173126615,
+      "grad_norm": 1.4743701219558716,
+      "learning_rate": 4.46264367816092e-06,
+      "loss": 0.9811,
+      "step": 397
+    },
+    {
+      "epoch": 0.41136950904392766,
+      "grad_norm": 1.2035115957260132,
+      "learning_rate": 4.459770114942529e-06,
+      "loss": 0.9673,
+      "step": 398
+    },
+    {
+      "epoch": 0.4124031007751938,
+      "grad_norm": 1.3029043674468994,
+      "learning_rate": 4.456896551724138e-06,
+      "loss": 0.9564,
+      "step": 399
+    },
+    {
+      "epoch": 0.4134366925064599,
+      "grad_norm": 1.3427908420562744,
+      "learning_rate": 4.454022988505747e-06,
+      "loss": 0.9982,
+      "step": 400
+    },
+    {
+      "epoch": 0.4134366925064599,
+      "eval_loss": 1.091240644454956,
+      "eval_runtime": 52.5565,
+      "eval_samples_per_second": 19.027,
+      "eval_steps_per_second": 0.609,
+      "step": 400
+    },
+    {
+      "epoch": 0.4144702842377261,
+      "grad_norm": 1.3828445672988892,
+      "learning_rate": 4.4511494252873564e-06,
+      "loss": 0.9612,
+      "step": 401
+    },
+    {
+      "epoch": 0.41550387596899224,
+      "grad_norm": 1.3622734546661377,
+      "learning_rate": 4.4482758620689656e-06,
+      "loss": 1.0141,
+      "step": 402
+    },
+    {
+      "epoch": 0.4165374677002584,
+      "grad_norm": 1.2798258066177368,
+      "learning_rate": 4.4454022988505756e-06,
+      "loss": 0.9491,
+      "step": 403
+    },
+    {
+      "epoch": 0.41757105943152456,
+      "grad_norm": 1.223121166229248,
+      "learning_rate": 4.442528735632185e-06,
+      "loss": 0.9722,
+      "step": 404
+    },
+    {
+      "epoch": 0.4186046511627907,
+      "grad_norm": 1.2730870246887207,
+      "learning_rate": 4.439655172413794e-06,
+      "loss": 0.9465,
+      "step": 405
+    },
+    {
+      "epoch": 0.4196382428940568,
+      "grad_norm": 1.1687973737716675,
+      "learning_rate": 4.436781609195402e-06,
+      "loss": 0.9508,
+      "step": 406
+    },
+    {
+      "epoch": 0.420671834625323,
+      "grad_norm": 1.233321189880371,
+      "learning_rate": 4.433908045977012e-06,
+      "loss": 0.9406,
+      "step": 407
+    },
+    {
+      "epoch": 0.42170542635658914,
+      "grad_norm": 1.6981616020202637,
+      "learning_rate": 4.431034482758621e-06,
+      "loss": 0.9796,
+      "step": 408
+    },
+    {
+      "epoch": 0.4227390180878553,
+      "grad_norm": 2.0248169898986816,
+      "learning_rate": 4.4281609195402304e-06,
+      "loss": 1.0083,
+      "step": 409
+    },
+    {
+      "epoch": 0.42377260981912146,
+      "grad_norm": 1.5073792934417725,
+      "learning_rate": 4.42528735632184e-06,
+      "loss": 1.0211,
+      "step": 410
+    },
+    {
+      "epoch": 0.4248062015503876,
+      "grad_norm": 1.2407021522521973,
+      "learning_rate": 4.422413793103449e-06,
+      "loss": 0.965,
+      "step": 411
+    },
+    {
+      "epoch": 0.4258397932816537,
+      "grad_norm": 1.5259541273117065,
+      "learning_rate": 4.419540229885058e-06,
+      "loss": 0.9618,
+      "step": 412
+    },
+    {
+      "epoch": 0.4268733850129199,
+      "grad_norm": 1.3160191774368286,
+      "learning_rate": 4.416666666666667e-06,
+      "loss": 1.0031,
+      "step": 413
+    },
+    {
+      "epoch": 0.42790697674418604,
+      "grad_norm": 1.366034984588623,
+      "learning_rate": 4.413793103448276e-06,
+      "loss": 0.9726,
+      "step": 414
+    },
+    {
+      "epoch": 0.4289405684754522,
+      "grad_norm": 1.3648245334625244,
+      "learning_rate": 4.410919540229885e-06,
+      "loss": 0.9727,
+      "step": 415
+    },
+    {
+      "epoch": 0.42997416020671836,
+      "grad_norm": 1.4371240139007568,
+      "learning_rate": 4.4080459770114944e-06,
+      "loss": 0.9921,
+      "step": 416
+    },
+    {
+      "epoch": 0.4310077519379845,
+      "grad_norm": 1.4490691423416138,
+      "learning_rate": 4.405172413793104e-06,
+      "loss": 1.0286,
+      "step": 417
+    },
+    {
+      "epoch": 0.4320413436692506,
+      "grad_norm": 1.4341950416564941,
+      "learning_rate": 4.402298850574713e-06,
+      "loss": 0.9953,
+      "step": 418
+    },
+    {
+      "epoch": 0.4330749354005168,
+      "grad_norm": 1.4683347940444946,
+      "learning_rate": 4.399425287356322e-06,
+      "loss": 0.9812,
+      "step": 419
+    },
+    {
+      "epoch": 0.43410852713178294,
+      "grad_norm": 1.459459662437439,
+      "learning_rate": 4.396551724137931e-06,
+      "loss": 0.9855,
+      "step": 420
+    },
+    {
+      "epoch": 0.4351421188630491,
+      "grad_norm": 1.3102967739105225,
+      "learning_rate": 4.39367816091954e-06,
+      "loss": 1.0135,
+      "step": 421
+    },
+    {
+      "epoch": 0.43617571059431526,
+      "grad_norm": 1.4162970781326294,
+      "learning_rate": 4.39080459770115e-06,
+      "loss": 1.0225,
+      "step": 422
+    },
+    {
+      "epoch": 0.4372093023255814,
+      "grad_norm": 1.4849135875701904,
+      "learning_rate": 4.387931034482759e-06,
+      "loss": 0.9791,
+      "step": 423
+    },
+    {
+      "epoch": 0.4382428940568475,
+      "grad_norm": 1.3206710815429688,
+      "learning_rate": 4.3850574712643685e-06,
+      "loss": 0.9568,
+      "step": 424
+    },
+    {
+      "epoch": 0.4392764857881137,
+      "grad_norm": 1.401818037033081,
+      "learning_rate": 4.382183908045977e-06,
+      "loss": 1.0067,
+      "step": 425
+    },
+    {
+      "epoch": 0.44031007751937984,
+      "grad_norm": 1.4235022068023682,
+      "learning_rate": 4.379310344827587e-06,
+      "loss": 0.9687,
+      "step": 426
+    },
+    {
+      "epoch": 0.441343669250646,
+      "grad_norm": 1.413816213607788,
+      "learning_rate": 4.376436781609196e-06,
+      "loss": 0.9889,
+      "step": 427
+    },
+    {
+      "epoch": 0.44237726098191216,
+      "grad_norm": 1.4104002714157104,
+      "learning_rate": 4.373563218390805e-06,
+      "loss": 1.0102,
+      "step": 428
+    },
+    {
+      "epoch": 0.4434108527131783,
+      "grad_norm": 1.1682835817337036,
+      "learning_rate": 4.370689655172414e-06,
+      "loss": 0.9593,
+      "step": 429
+    },
+    {
+      "epoch": 0.4444444444444444,
+      "grad_norm": 1.4189906120300293,
+      "learning_rate": 4.367816091954023e-06,
+      "loss": 0.9598,
+      "step": 430
+    },
+    {
+      "epoch": 0.4454780361757106,
+      "grad_norm": 1.4035252332687378,
+      "learning_rate": 4.3649425287356325e-06,
+      "loss": 0.9726,
+      "step": 431
+    },
+    {
+      "epoch": 0.44651162790697674,
+      "grad_norm": 1.3095788955688477,
+      "learning_rate": 4.362068965517242e-06,
+      "loss": 0.9867,
+      "step": 432
+    },
+    {
+      "epoch": 0.4475452196382429,
+      "grad_norm": 1.5546255111694336,
+      "learning_rate": 4.359195402298851e-06,
+      "loss": 0.9851,
+      "step": 433
+    },
+    {
+      "epoch": 0.44857881136950906,
+      "grad_norm": 1.545753002166748,
+      "learning_rate": 4.35632183908046e-06,
+      "loss": 1.002,
+      "step": 434
+    },
+    {
+      "epoch": 0.4496124031007752,
+      "grad_norm": 1.5164129734039307,
+      "learning_rate": 4.353448275862069e-06,
+      "loss": 0.9835,
+      "step": 435
+    },
+    {
+      "epoch": 0.4506459948320413,
+      "grad_norm": 1.285075306892395,
+      "learning_rate": 4.350574712643679e-06,
+      "loss": 0.991,
+      "step": 436
+    },
+    {
+      "epoch": 0.4516795865633075,
+      "grad_norm": 1.2409052848815918,
+      "learning_rate": 4.347701149425288e-06,
+      "loss": 0.9413,
+      "step": 437
+    },
+    {
+      "epoch": 0.45271317829457364,
+      "grad_norm": 1.6471928358078003,
+      "learning_rate": 4.3448275862068965e-06,
+      "loss": 0.9642,
+      "step": 438
+    },
+    {
+      "epoch": 0.4537467700258398,
+      "grad_norm": 1.3383151292800903,
+      "learning_rate": 4.341954022988506e-06,
+      "loss": 0.9621,
+      "step": 439
+    },
+    {
+      "epoch": 0.45478036175710596,
+      "grad_norm": 1.2632237672805786,
+      "learning_rate": 4.339080459770116e-06,
+      "loss": 0.9729,
+      "step": 440
+    },
+    {
+      "epoch": 0.4558139534883721,
+      "grad_norm": 1.1937509775161743,
+      "learning_rate": 4.336206896551725e-06,
+      "loss": 0.987,
+      "step": 441
+    },
+    {
+      "epoch": 0.4568475452196382,
+      "grad_norm": 1.5322760343551636,
+      "learning_rate": 4.333333333333334e-06,
+      "loss": 0.9693,
+      "step": 442
+    },
+    {
+      "epoch": 0.4578811369509044,
+      "grad_norm": 1.3935906887054443,
+      "learning_rate": 4.330459770114943e-06,
+      "loss": 0.9734,
+      "step": 443
+    },
+    {
+      "epoch": 0.45891472868217054,
+      "grad_norm": 1.305679440498352,
+      "learning_rate": 4.327586206896552e-06,
+      "loss": 0.9886,
+      "step": 444
+    },
+    {
+      "epoch": 0.4599483204134367,
+      "grad_norm": 1.2530243396759033,
+      "learning_rate": 4.324712643678161e-06,
+      "loss": 1.0004,
+      "step": 445
+    },
+    {
+      "epoch": 0.46098191214470285,
+      "grad_norm": 1.4160491228103638,
+      "learning_rate": 4.3218390804597705e-06,
+      "loss": 0.9865,
+      "step": 446
+    },
+    {
+      "epoch": 0.462015503875969,
+      "grad_norm": 1.4170372486114502,
+      "learning_rate": 4.31896551724138e-06,
+      "loss": 0.9965,
+      "step": 447
+    },
+    {
+      "epoch": 0.4630490956072351,
+      "grad_norm": 1.4800901412963867,
+      "learning_rate": 4.316091954022989e-06,
+      "loss": 0.9386,
+      "step": 448
+    },
+    {
+      "epoch": 0.4640826873385013,
+      "grad_norm": 1.3258070945739746,
+      "learning_rate": 4.313218390804598e-06,
+      "loss": 0.9501,
+      "step": 449
+    },
+    {
+      "epoch": 0.46511627906976744,
+      "grad_norm": 1.2615549564361572,
+      "learning_rate": 4.310344827586207e-06,
+      "loss": 0.9698,
+      "step": 450
+    },
+    {
+      "epoch": 0.46511627906976744,
+      "eval_loss": 1.0821963548660278,
+      "eval_runtime": 52.7658,
+      "eval_samples_per_second": 18.952,
+      "eval_steps_per_second": 0.606,
+      "step": 450
+    },
+    {
+      "epoch": 0.4661498708010336,
+      "grad_norm": 1.5667704343795776,
+      "learning_rate": 4.307471264367816e-06,
+      "loss": 0.9839,
+      "step": 451
+    },
+    {
+      "epoch": 0.46718346253229975,
+      "grad_norm": 1.287380576133728,
+      "learning_rate": 4.304597701149425e-06,
+      "loss": 0.9428,
+      "step": 452
+    },
+    {
+      "epoch": 0.4682170542635659,
+      "grad_norm": 1.3471415042877197,
+      "learning_rate": 4.3017241379310345e-06,
+      "loss": 0.9671,
+      "step": 453
+    },
+    {
+      "epoch": 0.469250645994832,
+      "grad_norm": 1.1595243215560913,
+      "learning_rate": 4.2988505747126445e-06,
+      "loss": 0.9905,
+      "step": 454
+    },
+    {
+      "epoch": 0.4702842377260982,
+      "grad_norm": 1.6918305158615112,
+      "learning_rate": 4.295977011494254e-06,
+      "loss": 0.9637,
+      "step": 455
+    },
+    {
+      "epoch": 0.47131782945736433,
+      "grad_norm": 1.4655194282531738,
+      "learning_rate": 4.293103448275863e-06,
+      "loss": 0.9627,
+      "step": 456
+    },
+    {
+      "epoch": 0.4723514211886305,
+      "grad_norm": 1.2935839891433716,
+      "learning_rate": 4.290229885057471e-06,
+      "loss": 0.9736,
+      "step": 457
+    },
+    {
+      "epoch": 0.47338501291989665,
+      "grad_norm": 1.3914062976837158,
+      "learning_rate": 4.287356321839081e-06,
+      "loss": 0.9054,
+      "step": 458
+    },
+    {
+      "epoch": 0.4744186046511628,
+      "grad_norm": 1.7281622886657715,
+      "learning_rate": 4.28448275862069e-06,
+      "loss": 0.9869,
+      "step": 459
+    },
+    {
+      "epoch": 0.4754521963824289,
+      "grad_norm": 1.552720069885254,
+      "learning_rate": 4.281609195402299e-06,
+      "loss": 0.9255,
+      "step": 460
+    },
+    {
+      "epoch": 0.4764857881136951,
+      "grad_norm": 1.1797876358032227,
+      "learning_rate": 4.2787356321839085e-06,
+      "loss": 0.948,
+      "step": 461
+    },
+    {
+      "epoch": 0.47751937984496123,
+      "grad_norm": 1.426317811012268,
+      "learning_rate": 4.275862068965518e-06,
+      "loss": 0.9781,
+      "step": 462
+    },
+    {
+      "epoch": 0.4785529715762274,
+      "grad_norm": 1.3194639682769775,
+      "learning_rate": 4.272988505747127e-06,
+      "loss": 1.0075,
+      "step": 463
+    },
+    {
+      "epoch": 0.47958656330749355,
+      "grad_norm": 1.493377447128296,
+      "learning_rate": 4.270114942528736e-06,
+      "loss": 0.9497,
+      "step": 464
+    },
+    {
+      "epoch": 0.4806201550387597,
+      "grad_norm": 1.4328482151031494,
+      "learning_rate": 4.267241379310345e-06,
+      "loss": 0.9838,
+      "step": 465
+    },
+    {
+      "epoch": 0.4816537467700258,
+      "grad_norm": 1.420423984527588,
+      "learning_rate": 4.264367816091954e-06,
+      "loss": 0.9939,
+      "step": 466
+    },
+    {
+      "epoch": 0.482687338501292,
+      "grad_norm": 1.3972870111465454,
+      "learning_rate": 4.261494252873563e-06,
+      "loss": 1.0312,
+      "step": 467
+    },
+    {
+      "epoch": 0.48372093023255813,
+      "grad_norm": 1.2692211866378784,
+      "learning_rate": 4.2586206896551725e-06,
+      "loss": 0.9715,
+      "step": 468
+    },
+    {
+      "epoch": 0.4847545219638243,
+      "grad_norm": 1.4559839963912964,
+      "learning_rate": 4.2557471264367825e-06,
+      "loss": 1.0095,
+      "step": 469
+    },
+    {
+      "epoch": 0.48578811369509045,
+      "grad_norm": 1.3825651407241821,
+      "learning_rate": 4.252873563218391e-06,
+      "loss": 0.9871,
+      "step": 470
+    },
+    {
+      "epoch": 0.4868217054263566,
+      "grad_norm": 1.1747273206710815,
+      "learning_rate": 4.25e-06,
+      "loss": 0.9756,
+      "step": 471
+    },
+    {
+      "epoch": 0.4878552971576227,
+      "grad_norm": 1.3212231397628784,
+      "learning_rate": 4.247126436781609e-06,
+      "loss": 0.9916,
+      "step": 472
+    },
+    {
+      "epoch": 0.4888888888888889,
+      "grad_norm": 1.4103412628173828,
+      "learning_rate": 4.244252873563219e-06,
+      "loss": 0.9748,
+      "step": 473
+    },
+    {
+      "epoch": 0.48992248062015503,
+      "grad_norm": 1.3848503828048706,
+      "learning_rate": 4.241379310344828e-06,
+      "loss": 0.941,
+      "step": 474
+    },
+    {
+      "epoch": 0.4909560723514212,
+      "grad_norm": 1.3593331575393677,
+      "learning_rate": 4.238505747126437e-06,
+      "loss": 0.9571,
+      "step": 475
+    },
+    {
+      "epoch": 0.49198966408268735,
+      "grad_norm": 1.3448289632797241,
+      "learning_rate": 4.235632183908046e-06,
+      "loss": 0.9889,
+      "step": 476
+    },
+    {
+      "epoch": 0.4930232558139535,
+      "grad_norm": 1.2616806030273438,
+      "learning_rate": 4.232758620689656e-06,
+      "loss": 0.9372,
+      "step": 477
+    },
+    {
+      "epoch": 0.4940568475452196,
+      "grad_norm": 1.2876821756362915,
+      "learning_rate": 4.229885057471265e-06,
+      "loss": 0.9688,
+      "step": 478
+    },
+    {
+      "epoch": 0.49509043927648577,
+      "grad_norm": 1.2750991582870483,
+      "learning_rate": 4.227011494252874e-06,
+      "loss": 0.9564,
+      "step": 479
+    },
+    {
+      "epoch": 0.49612403100775193,
+      "grad_norm": 1.4402563571929932,
+      "learning_rate": 4.224137931034483e-06,
+      "loss": 0.9376,
+      "step": 480
+    },
+    {
+      "epoch": 0.4971576227390181,
+      "grad_norm": 1.5290478467941284,
+      "learning_rate": 4.221264367816092e-06,
+      "loss": 0.9947,
+      "step": 481
+    },
+    {
+      "epoch": 0.49819121447028425,
+      "grad_norm": 1.6395635604858398,
+      "learning_rate": 4.218390804597701e-06,
+      "loss": 0.9764,
+      "step": 482
+    },
+    {
+      "epoch": 0.4992248062015504,
+      "grad_norm": 1.3985153436660767,
+      "learning_rate": 4.2155172413793106e-06,
+      "loss": 0.9296,
+      "step": 483
+    },
+    {
+      "epoch": 0.5002583979328166,
+      "grad_norm": 1.1870092153549194,
+      "learning_rate": 4.21264367816092e-06,
+      "loss": 0.9223,
+      "step": 484
+    },
+    {
+      "epoch": 0.5012919896640827,
+      "grad_norm": 1.3690369129180908,
+      "learning_rate": 4.209770114942529e-06,
+      "loss": 0.9249,
+      "step": 485
+    },
+    {
+      "epoch": 0.5023255813953489,
+      "grad_norm": 1.1812691688537598,
+      "learning_rate": 4.206896551724138e-06,
+      "loss": 0.9863,
+      "step": 486
+    },
+    {
+      "epoch": 0.5033591731266149,
+      "grad_norm": 1.2263563871383667,
+      "learning_rate": 4.204022988505748e-06,
+      "loss": 0.9696,
+      "step": 487
+    },
+    {
+      "epoch": 0.5043927648578811,
+      "grad_norm": 1.3079935312271118,
+      "learning_rate": 4.201149425287357e-06,
+      "loss": 0.9546,
+      "step": 488
+    },
+    {
+      "epoch": 0.5054263565891473,
+      "grad_norm": 1.288635015487671,
+      "learning_rate": 4.1982758620689654e-06,
+      "loss": 0.9735,
+      "step": 489
+    },
+    {
+      "epoch": 0.5064599483204134,
+      "grad_norm": 1.3041417598724365,
+      "learning_rate": 4.1954022988505746e-06,
+      "loss": 0.9615,
+      "step": 490
+    },
+    {
+      "epoch": 0.5074935400516796,
+      "grad_norm": 1.2297852039337158,
+      "learning_rate": 4.1925287356321846e-06,
+      "loss": 1.0156,
+      "step": 491
+    },
+    {
+      "epoch": 0.5085271317829457,
+      "grad_norm": 1.167002558708191,
+      "learning_rate": 4.189655172413794e-06,
+      "loss": 0.9535,
+      "step": 492
+    },
+    {
+      "epoch": 0.5095607235142119,
+      "grad_norm": 1.1384843587875366,
+      "learning_rate": 4.186781609195403e-06,
+      "loss": 0.9567,
+      "step": 493
+    },
+    {
+      "epoch": 0.510594315245478,
+      "grad_norm": 1.6105479001998901,
+      "learning_rate": 4.183908045977012e-06,
+      "loss": 1.0121,
+      "step": 494
+    },
+    {
+      "epoch": 0.5116279069767442,
+      "grad_norm": 1.397196888923645,
+      "learning_rate": 4.181034482758621e-06,
+      "loss": 0.953,
+      "step": 495
+    },
+    {
+      "epoch": 0.5126614987080104,
+      "grad_norm": 1.5293458700180054,
+      "learning_rate": 4.17816091954023e-06,
+      "loss": 0.9612,
+      "step": 496
+    },
+    {
+      "epoch": 0.5136950904392765,
+      "grad_norm": 1.6168063879013062,
+      "learning_rate": 4.1752873563218394e-06,
+      "loss": 0.9967,
+      "step": 497
+    },
+    {
+      "epoch": 0.5147286821705427,
+      "grad_norm": 1.3672846555709839,
+      "learning_rate": 4.1724137931034486e-06,
+      "loss": 0.9388,
+      "step": 498
+    },
+    {
+      "epoch": 0.5157622739018087,
+      "grad_norm": 1.4402540922164917,
+      "learning_rate": 4.169540229885058e-06,
+      "loss": 0.9299,
+      "step": 499
+    },
+    {
+      "epoch": 0.5167958656330749,
+      "grad_norm": 1.5125951766967773,
+      "learning_rate": 4.166666666666667e-06,
+      "loss": 0.9761,
+      "step": 500
+    },
+    {
+      "epoch": 0.5167958656330749,
+      "eval_loss": 1.0677101612091064,
+      "eval_runtime": 52.5704,
+      "eval_samples_per_second": 19.022,
+      "eval_steps_per_second": 0.609,
+      "step": 500
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1934,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.87708552822784e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_control_10M_seed0/runs/checkpoint-500/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/blobs/7c8139ab9a4e680ff0e9741c678e26c43788abf0

@@ -0,0 +1,586 @@
+{
+  "metadata": {
+    "total_size": 29540067328
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00008-of-00008.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.40.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00008-of-00008.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00008-of-00008.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00008-of-00008.safetensors",
+    "model.layers.47.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.norm.weight": "model-00008-of-00008.safetensors"
+  }
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/blobs/83bba6e26639fb152bd0077977cf6ea8312b42a9

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 13824,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 48,
+  "model_type": "qwen2",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/blobs/cbbb3133034e192527e5321b4c679154e4819ab8

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": false,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.37.0"
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/refs/main

@@ -0,0 +1 @@
+97e1e76335b7017d8f67c08a19d103c0504298c9

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/snapshots/97e1e76335b7017d8f67c08a19d103c0504298c9/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 13824,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 48,
+  "model_type": "qwen2",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/snapshots/97e1e76335b7017d8f67c08a19d103c0504298c9/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": false,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.37.0"
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/artifacts/models--Qwen--Qwen2.5-14B/snapshots/97e1e76335b7017d8f67c08a19d103c0504298c9/model.safetensors.index.json

@@ -0,0 +1,586 @@
+{
+  "metadata": {
+    "total_size": 29540067328
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00008-of-00008.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00008.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00008.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00004-of-00008.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00008.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00005-of-00008.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00005-of-00008.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00008.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00008.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.40.self_attn.k_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.q_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.v_proj.bias": "model-00006-of-00008.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00006-of-00008.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00008-of-00008.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00008-of-00008.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00008-of-00008.safetensors",
+    "model.layers.47.self_attn.k_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.q_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.v_proj.bias": "model-00007-of-00008.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00007-of-00008.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00008.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00008.safetensors",
+    "model.norm.weight": "model-00008-of-00008.safetensors"
+  }
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1934/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1934/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-14B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 13824,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 48,
+  "model_type": "qwen2",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1934/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.45.1"
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1934/latest

@@ -0,0 +1 @@
+global_step1934

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1934/model.safetensors.index.json

@@ -0,0 +1,586 @@
+{
+  "metadata": {
+    "total_size": 29540067328
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.42.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.k_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.q_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.v_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.k_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.q_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.v_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.k_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.q_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.v_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.k_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.q_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.v_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.k_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.q_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.v_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1934/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1934/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1934/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-500/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-500/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-14B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 13824,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 48,
+  "model_type": "qwen2",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-500/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.45.1"
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-500/latest

@@ -0,0 +1 @@
+global_step500

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-500/model.safetensors.index.json

@@ -0,0 +1,586 @@
+{
+  "metadata": {
+    "total_size": 29540067328
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00004-of-00006.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.42.self_attn.k_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.q_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.v_proj.bias": "model-00005-of-00006.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.k_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.q_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.v_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.k_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.q_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.v_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.k_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.q_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.v_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.k_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.q_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.v_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.k_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.q_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.v_proj.bias": "model-00006-of-00006.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-500/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-500/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/Qwen2.5-14B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-500/trainer_state.json

@@ -0,0 +1,3613 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5167958656330749,
+  "eval_steps": 50,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0010335917312661498,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7469,
+      "step": 1
+    },
+    {
+      "epoch": 0.0020671834625322996,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7373,
+      "step": 2
+    },
+    {
+      "epoch": 0.0031007751937984496,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7672,
+      "step": 3
+    },
+    {
+      "epoch": 0.004134366925064599,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7292,
+      "step": 4
+    },
+    {
+      "epoch": 0.00516795865633075,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7527,
+      "step": 5
+    },
+    {
+      "epoch": 0.006201550387596899,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.764,
+      "step": 6
+    },
+    {
+      "epoch": 0.007235142118863049,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7631,
+      "step": 7
+    },
+    {
+      "epoch": 0.008268733850129198,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.743,
+      "step": 8
+    },
+    {
+      "epoch": 0.009302325581395349,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7658,
+      "step": 9
+    },
+    {
+      "epoch": 0.0103359173126615,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7763,
+      "step": 10
+    },
+    {
+      "epoch": 0.011369509043927648,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7739,
+      "step": 11
+    },
+    {
+      "epoch": 0.012403100775193798,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7558,
+      "step": 12
+    },
+    {
+      "epoch": 0.013436692506459949,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7536,
+      "step": 13
+    },
+    {
+      "epoch": 0.014470284237726097,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7285,
+      "step": 14
+    },
+    {
+      "epoch": 0.015503875968992248,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.7508,
+      "step": 15
+    },
+    {
+      "epoch": 0.016537467700258397,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 1.753,
+      "step": 16
+    },
+    {
+      "epoch": 0.01757105943152455,
+      "grad_norm": 3.294677734375,
+      "learning_rate": 2.5773195876288662e-08,
+      "loss": 1.7609,
+      "step": 17
+    },
+    {
+      "epoch": 0.018604651162790697,
+      "grad_norm": 3.3554086685180664,
+      "learning_rate": 5.1546391752577325e-08,
+      "loss": 1.7881,
+      "step": 18
+    },
+    {
+      "epoch": 0.019638242894056846,
+      "grad_norm": 2.9532411098480225,
+      "learning_rate": 7.731958762886598e-08,
+      "loss": 1.7593,
+      "step": 19
+    },
+    {
+      "epoch": 0.020671834625323,
+      "grad_norm": 2.7428970336914062,
+      "learning_rate": 1.0309278350515465e-07,
+      "loss": 1.7514,
+      "step": 20
+    },
+    {
+      "epoch": 0.021705426356589147,
+      "grad_norm": 2.5760021209716797,
+      "learning_rate": 1.288659793814433e-07,
+      "loss": 1.7519,
+      "step": 21
+    },
+    {
+      "epoch": 0.022739018087855296,
+      "grad_norm": 3.8851864337921143,
+      "learning_rate": 1.5463917525773197e-07,
+      "loss": 1.7453,
+      "step": 22
+    },
+    {
+      "epoch": 0.023772609819121448,
+      "grad_norm": 3.3514232635498047,
+      "learning_rate": 1.804123711340206e-07,
+      "loss": 1.7524,
+      "step": 23
+    },
+    {
+      "epoch": 0.024806201550387597,
+      "grad_norm": 3.9262187480926514,
+      "learning_rate": 2.061855670103093e-07,
+      "loss": 1.7754,
+      "step": 24
+    },
+    {
+      "epoch": 0.025839793281653745,
+      "grad_norm": 3.5925371646881104,
+      "learning_rate": 2.3195876288659797e-07,
+      "loss": 1.7711,
+      "step": 25
+    },
+    {
+      "epoch": 0.026873385012919897,
+      "grad_norm": 3.2370054721832275,
+      "learning_rate": 2.577319587628866e-07,
+      "loss": 1.7327,
+      "step": 26
+    },
+    {
+      "epoch": 0.027906976744186046,
+      "grad_norm": 2.7215564250946045,
+      "learning_rate": 2.8350515463917527e-07,
+      "loss": 1.7457,
+      "step": 27
+    },
+    {
+      "epoch": 0.028940568475452195,
+      "grad_norm": 3.2791647911071777,
+      "learning_rate": 3.0927835051546394e-07,
+      "loss": 1.7645,
+      "step": 28
+    },
+    {
+      "epoch": 0.029974160206718347,
+      "grad_norm": 2.451993942260742,
+      "learning_rate": 3.350515463917526e-07,
+      "loss": 1.7544,
+      "step": 29
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 2.290065288543701,
+      "learning_rate": 3.608247422680412e-07,
+      "loss": 1.7698,
+      "step": 30
+    },
+    {
+      "epoch": 0.03204134366925065,
+      "grad_norm": 1.9393961429595947,
+      "learning_rate": 3.8659793814432993e-07,
+      "loss": 1.7214,
+      "step": 31
+    },
+    {
+      "epoch": 0.03307493540051679,
+      "grad_norm": 2.158582925796509,
+      "learning_rate": 4.123711340206186e-07,
+      "loss": 1.7211,
+      "step": 32
+    },
+    {
+      "epoch": 0.034108527131782945,
+      "grad_norm": 1.6859667301177979,
+      "learning_rate": 4.381443298969072e-07,
+      "loss": 1.7438,
+      "step": 33
+    },
+    {
+      "epoch": 0.0351421188630491,
+      "grad_norm": 1.663550615310669,
+      "learning_rate": 4.6391752577319593e-07,
+      "loss": 1.7394,
+      "step": 34
+    },
+    {
+      "epoch": 0.03617571059431524,
+      "grad_norm": 1.6059417724609375,
+      "learning_rate": 4.896907216494846e-07,
+      "loss": 1.7381,
+      "step": 35
+    },
+    {
+      "epoch": 0.037209302325581395,
+      "grad_norm": 1.9995272159576416,
+      "learning_rate": 5.154639175257732e-07,
+      "loss": 1.7384,
+      "step": 36
+    },
+    {
+      "epoch": 0.03824289405684755,
+      "grad_norm": 2.1068830490112305,
+      "learning_rate": 5.412371134020619e-07,
+      "loss": 1.7148,
+      "step": 37
+    },
+    {
+      "epoch": 0.03927648578811369,
+      "grad_norm": 1.5573288202285767,
+      "learning_rate": 5.670103092783505e-07,
+      "loss": 1.6775,
+      "step": 38
+    },
+    {
+      "epoch": 0.040310077519379844,
+      "grad_norm": 1.5803656578063965,
+      "learning_rate": 5.927835051546392e-07,
+      "loss": 1.7361,
+      "step": 39
+    },
+    {
+      "epoch": 0.041343669250646,
+      "grad_norm": 1.535320520401001,
+      "learning_rate": 6.185567010309279e-07,
+      "loss": 1.7238,
+      "step": 40
+    },
+    {
+      "epoch": 0.04237726098191214,
+      "grad_norm": 1.4646360874176025,
+      "learning_rate": 6.443298969072165e-07,
+      "loss": 1.7127,
+      "step": 41
+    },
+    {
+      "epoch": 0.043410852713178294,
+      "grad_norm": 1.7842413187026978,
+      "learning_rate": 6.701030927835052e-07,
+      "loss": 1.7335,
+      "step": 42
+    },
+    {
+      "epoch": 0.044444444444444446,
+      "grad_norm": 1.287598967552185,
+      "learning_rate": 6.958762886597939e-07,
+      "loss": 1.681,
+      "step": 43
+    },
+    {
+      "epoch": 0.04547803617571059,
+      "grad_norm": 1.4461113214492798,
+      "learning_rate": 7.216494845360824e-07,
+      "loss": 1.6995,
+      "step": 44
+    },
+    {
+      "epoch": 0.046511627906976744,
+      "grad_norm": 1.3189141750335693,
+      "learning_rate": 7.474226804123711e-07,
+      "loss": 1.6825,
+      "step": 45
+    },
+    {
+      "epoch": 0.047545219638242896,
+      "grad_norm": 1.305091142654419,
+      "learning_rate": 7.731958762886599e-07,
+      "loss": 1.6656,
+      "step": 46
+    },
+    {
+      "epoch": 0.04857881136950904,
+      "grad_norm": 1.203321099281311,
+      "learning_rate": 7.989690721649485e-07,
+      "loss": 1.6695,
+      "step": 47
+    },
+    {
+      "epoch": 0.04961240310077519,
+      "grad_norm": 1.3265272378921509,
+      "learning_rate": 8.247422680412372e-07,
+      "loss": 1.6644,
+      "step": 48
+    },
+    {
+      "epoch": 0.050645994832041345,
+      "grad_norm": 1.613187313079834,
+      "learning_rate": 8.505154639175259e-07,
+      "loss": 1.673,
+      "step": 49
+    },
+    {
+      "epoch": 0.05167958656330749,
+      "grad_norm": 1.8446052074432373,
+      "learning_rate": 8.762886597938144e-07,
+      "loss": 1.6362,
+      "step": 50
+    },
+    {
+      "epoch": 0.05167958656330749,
+      "eval_loss": 1.6590594053268433,
+      "eval_runtime": 47.8421,
+      "eval_samples_per_second": 20.902,
+      "eval_steps_per_second": 0.669,
+      "step": 50
+    },
+    {
+      "epoch": 0.05271317829457364,
+      "grad_norm": 1.5133100748062134,
+      "learning_rate": 9.020618556701031e-07,
+      "loss": 1.6278,
+      "step": 51
+    },
+    {
+      "epoch": 0.053746770025839795,
+      "grad_norm": 1.6770209074020386,
+      "learning_rate": 9.278350515463919e-07,
+      "loss": 1.6379,
+      "step": 52
+    },
+    {
+      "epoch": 0.05478036175710594,
+      "grad_norm": 1.5879249572753906,
+      "learning_rate": 9.536082474226805e-07,
+      "loss": 1.6203,
+      "step": 53
+    },
+    {
+      "epoch": 0.05581395348837209,
+      "grad_norm": 2.090453863143921,
+      "learning_rate": 9.793814432989692e-07,
+      "loss": 1.6323,
+      "step": 54
+    },
+    {
+      "epoch": 0.056847545219638244,
+      "grad_norm": 1.4961861371994019,
+      "learning_rate": 1.005154639175258e-06,
+      "loss": 1.5958,
+      "step": 55
+    },
+    {
+      "epoch": 0.05788113695090439,
+      "grad_norm": 1.6027110815048218,
+      "learning_rate": 1.0309278350515464e-06,
+      "loss": 1.5987,
+      "step": 56
+    },
+    {
+      "epoch": 0.05891472868217054,
+      "grad_norm": 1.8416661024093628,
+      "learning_rate": 1.0567010309278351e-06,
+      "loss": 1.5536,
+      "step": 57
+    },
+    {
+      "epoch": 0.059948320413436694,
+      "grad_norm": 1.5269109010696411,
+      "learning_rate": 1.0824742268041239e-06,
+      "loss": 1.5569,
+      "step": 58
+    },
+    {
+      "epoch": 0.06098191214470284,
+      "grad_norm": 1.7397830486297607,
+      "learning_rate": 1.1082474226804124e-06,
+      "loss": 1.5512,
+      "step": 59
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 1.7923859357833862,
+      "learning_rate": 1.134020618556701e-06,
+      "loss": 1.5562,
+      "step": 60
+    },
+    {
+      "epoch": 0.06304909560723514,
+      "grad_norm": 1.4901388883590698,
+      "learning_rate": 1.1597938144329898e-06,
+      "loss": 1.5431,
+      "step": 61
+    },
+    {
+      "epoch": 0.0640826873385013,
+      "grad_norm": 1.6800209283828735,
+      "learning_rate": 1.1855670103092783e-06,
+      "loss": 1.5266,
+      "step": 62
+    },
+    {
+      "epoch": 0.06511627906976744,
+      "grad_norm": 1.9110485315322876,
+      "learning_rate": 1.211340206185567e-06,
+      "loss": 1.5346,
+      "step": 63
+    },
+    {
+      "epoch": 0.06614987080103359,
+      "grad_norm": 2.066371440887451,
+      "learning_rate": 1.2371134020618557e-06,
+      "loss": 1.5244,
+      "step": 64
+    },
+    {
+      "epoch": 0.06718346253229975,
+      "grad_norm": 1.4916744232177734,
+      "learning_rate": 1.2628865979381445e-06,
+      "loss": 1.516,
+      "step": 65
+    },
+    {
+      "epoch": 0.06821705426356589,
+      "grad_norm": 1.6237688064575195,
+      "learning_rate": 1.288659793814433e-06,
+      "loss": 1.5123,
+      "step": 66
+    },
+    {
+      "epoch": 0.06925064599483204,
+      "grad_norm": 1.643630027770996,
+      "learning_rate": 1.314432989690722e-06,
+      "loss": 1.4794,
+      "step": 67
+    },
+    {
+      "epoch": 0.0702842377260982,
+      "grad_norm": 1.4067258834838867,
+      "learning_rate": 1.3402061855670104e-06,
+      "loss": 1.4954,
+      "step": 68
+    },
+    {
+      "epoch": 0.07131782945736434,
+      "grad_norm": 1.4735355377197266,
+      "learning_rate": 1.365979381443299e-06,
+      "loss": 1.4949,
+      "step": 69
+    },
+    {
+      "epoch": 0.07235142118863049,
+      "grad_norm": 1.4811455011367798,
+      "learning_rate": 1.3917525773195878e-06,
+      "loss": 1.5264,
+      "step": 70
+    },
+    {
+      "epoch": 0.07338501291989664,
+      "grad_norm": 1.1310720443725586,
+      "learning_rate": 1.4175257731958764e-06,
+      "loss": 1.5018,
+      "step": 71
+    },
+    {
+      "epoch": 0.07441860465116279,
+      "grad_norm": 1.6779255867004395,
+      "learning_rate": 1.4432989690721649e-06,
+      "loss": 1.4662,
+      "step": 72
+    },
+    {
+      "epoch": 0.07545219638242893,
+      "grad_norm": 1.1733572483062744,
+      "learning_rate": 1.4690721649484538e-06,
+      "loss": 1.4903,
+      "step": 73
+    },
+    {
+      "epoch": 0.0764857881136951,
+      "grad_norm": 1.7176299095153809,
+      "learning_rate": 1.4948453608247423e-06,
+      "loss": 1.4544,
+      "step": 74
+    },
+    {
+      "epoch": 0.07751937984496124,
+      "grad_norm": 1.333526849746704,
+      "learning_rate": 1.520618556701031e-06,
+      "loss": 1.4476,
+      "step": 75
+    },
+    {
+      "epoch": 0.07855297157622738,
+      "grad_norm": 1.4169801473617554,
+      "learning_rate": 1.5463917525773197e-06,
+      "loss": 1.5011,
+      "step": 76
+    },
+    {
+      "epoch": 0.07958656330749354,
+      "grad_norm": 1.4367135763168335,
+      "learning_rate": 1.5721649484536082e-06,
+      "loss": 1.4157,
+      "step": 77
+    },
+    {
+      "epoch": 0.08062015503875969,
+      "grad_norm": 1.4957133531570435,
+      "learning_rate": 1.597938144329897e-06,
+      "loss": 1.4575,
+      "step": 78
+    },
+    {
+      "epoch": 0.08165374677002583,
+      "grad_norm": 1.357630729675293,
+      "learning_rate": 1.6237113402061857e-06,
+      "loss": 1.4428,
+      "step": 79
+    },
+    {
+      "epoch": 0.082687338501292,
+      "grad_norm": 1.884421706199646,
+      "learning_rate": 1.6494845360824744e-06,
+      "loss": 1.4447,
+      "step": 80
+    },
+    {
+      "epoch": 0.08372093023255814,
+      "grad_norm": 1.6526613235473633,
+      "learning_rate": 1.675257731958763e-06,
+      "loss": 1.4589,
+      "step": 81
+    },
+    {
+      "epoch": 0.08475452196382428,
+      "grad_norm": 1.6600714921951294,
+      "learning_rate": 1.7010309278350518e-06,
+      "loss": 1.4118,
+      "step": 82
+    },
+    {
+      "epoch": 0.08578811369509044,
+      "grad_norm": 1.6676090955734253,
+      "learning_rate": 1.7268041237113403e-06,
+      "loss": 1.419,
+      "step": 83
+    },
+    {
+      "epoch": 0.08682170542635659,
+      "grad_norm": 1.312245488166809,
+      "learning_rate": 1.7525773195876288e-06,
+      "loss": 1.4401,
+      "step": 84
+    },
+    {
+      "epoch": 0.08785529715762273,
+      "grad_norm": 1.5149967670440674,
+      "learning_rate": 1.7783505154639178e-06,
+      "loss": 1.4079,
+      "step": 85
+    },
+    {
+      "epoch": 0.08888888888888889,
+      "grad_norm": 1.487531065940857,
+      "learning_rate": 1.8041237113402063e-06,
+      "loss": 1.4028,
+      "step": 86
+    },
+    {
+      "epoch": 0.08992248062015504,
+      "grad_norm": 1.6969867944717407,
+      "learning_rate": 1.8298969072164948e-06,
+      "loss": 1.4423,
+      "step": 87
+    },
+    {
+      "epoch": 0.09095607235142118,
+      "grad_norm": 1.786713719367981,
+      "learning_rate": 1.8556701030927837e-06,
+      "loss": 1.4331,
+      "step": 88
+    },
+    {
+      "epoch": 0.09198966408268734,
+      "grad_norm": 1.43896484375,
+      "learning_rate": 1.8814432989690722e-06,
+      "loss": 1.4063,
+      "step": 89
+    },
+    {
+      "epoch": 0.09302325581395349,
+      "grad_norm": 2.0026865005493164,
+      "learning_rate": 1.907216494845361e-06,
+      "loss": 1.3993,
+      "step": 90
+    },
+    {
+      "epoch": 0.09405684754521963,
+      "grad_norm": 1.7065625190734863,
+      "learning_rate": 1.9329896907216497e-06,
+      "loss": 1.4029,
+      "step": 91
+    },
+    {
+      "epoch": 0.09509043927648579,
+      "grad_norm": 1.3610997200012207,
+      "learning_rate": 1.9587628865979384e-06,
+      "loss": 1.4246,
+      "step": 92
+    },
+    {
+      "epoch": 0.09612403100775194,
+      "grad_norm": 1.799842357635498,
+      "learning_rate": 1.9845360824742267e-06,
+      "loss": 1.4015,
+      "step": 93
+    },
+    {
+      "epoch": 0.09715762273901808,
+      "grad_norm": 1.0985530614852905,
+      "learning_rate": 2.010309278350516e-06,
+      "loss": 1.4481,
+      "step": 94
+    },
+    {
+      "epoch": 0.09819121447028424,
+      "grad_norm": 1.4739775657653809,
+      "learning_rate": 2.036082474226804e-06,
+      "loss": 1.4271,
+      "step": 95
+    },
+    {
+      "epoch": 0.09922480620155039,
+      "grad_norm": 1.3937931060791016,
+      "learning_rate": 2.061855670103093e-06,
+      "loss": 1.4002,
+      "step": 96
+    },
+    {
+      "epoch": 0.10025839793281653,
+      "grad_norm": 1.3285835981369019,
+      "learning_rate": 2.0876288659793816e-06,
+      "loss": 1.4201,
+      "step": 97
+    },
+    {
+      "epoch": 0.10129198966408269,
+      "grad_norm": 1.7354886531829834,
+      "learning_rate": 2.1134020618556703e-06,
+      "loss": 1.3842,
+      "step": 98
+    },
+    {
+      "epoch": 0.10232558139534884,
+      "grad_norm": 1.8828879594802856,
+      "learning_rate": 2.139175257731959e-06,
+      "loss": 1.3836,
+      "step": 99
+    },
+    {
+      "epoch": 0.10335917312661498,
+      "grad_norm": 1.6674432754516602,
+      "learning_rate": 2.1649484536082477e-06,
+      "loss": 1.3623,
+      "step": 100
+    },
+    {
+      "epoch": 0.10335917312661498,
+      "eval_loss": 1.4376535415649414,
+      "eval_runtime": 47.7079,
+      "eval_samples_per_second": 20.961,
+      "eval_steps_per_second": 0.671,
+      "step": 100
+    },
+    {
+      "epoch": 0.10439276485788114,
+      "grad_norm": 1.6396552324295044,
+      "learning_rate": 2.1907216494845364e-06,
+      "loss": 1.4139,
+      "step": 101
+    },
+    {
+      "epoch": 0.10542635658914729,
+      "grad_norm": 1.4648929834365845,
+      "learning_rate": 2.2164948453608247e-06,
+      "loss": 1.3518,
+      "step": 102
+    },
+    {
+      "epoch": 0.10645994832041343,
+      "grad_norm": 1.3640034198760986,
+      "learning_rate": 2.242268041237114e-06,
+      "loss": 1.368,
+      "step": 103
+    },
+    {
+      "epoch": 0.10749354005167959,
+      "grad_norm": 1.8199163675308228,
+      "learning_rate": 2.268041237113402e-06,
+      "loss": 1.3712,
+      "step": 104
+    },
+    {
+      "epoch": 0.10852713178294573,
+      "grad_norm": 1.469482421875,
+      "learning_rate": 2.293814432989691e-06,
+      "loss": 1.4095,
+      "step": 105
+    },
+    {
+      "epoch": 0.10956072351421188,
+      "grad_norm": 1.8030418157577515,
+      "learning_rate": 2.3195876288659796e-06,
+      "loss": 1.3821,
+      "step": 106
+    },
+    {
+      "epoch": 0.11059431524547804,
+      "grad_norm": 1.5513843297958374,
+      "learning_rate": 2.3453608247422683e-06,
+      "loss": 1.3813,
+      "step": 107
+    },
+    {
+      "epoch": 0.11162790697674418,
+      "grad_norm": 1.3880162239074707,
+      "learning_rate": 2.3711340206185566e-06,
+      "loss": 1.3776,
+      "step": 108
+    },
+    {
+      "epoch": 0.11266149870801033,
+      "grad_norm": 1.77285897731781,
+      "learning_rate": 2.3969072164948458e-06,
+      "loss": 1.4228,
+      "step": 109
+    },
+    {
+      "epoch": 0.11369509043927649,
+      "grad_norm": 1.7808395624160767,
+      "learning_rate": 2.422680412371134e-06,
+      "loss": 1.331,
+      "step": 110
+    },
+    {
+      "epoch": 0.11472868217054263,
+      "grad_norm": 1.3493247032165527,
+      "learning_rate": 2.4484536082474228e-06,
+      "loss": 1.3437,
+      "step": 111
+    },
+    {
+      "epoch": 0.11576227390180878,
+      "grad_norm": 1.8478275537490845,
+      "learning_rate": 2.4742268041237115e-06,
+      "loss": 1.3572,
+      "step": 112
+    },
+    {
+      "epoch": 0.11679586563307494,
+      "grad_norm": 1.654219627380371,
+      "learning_rate": 2.5e-06,
+      "loss": 1.3514,
+      "step": 113
+    },
+    {
+      "epoch": 0.11782945736434108,
+      "grad_norm": 2.001561164855957,
+      "learning_rate": 2.525773195876289e-06,
+      "loss": 1.3659,
+      "step": 114
+    },
+    {
+      "epoch": 0.11886304909560723,
+      "grad_norm": 1.4977449178695679,
+      "learning_rate": 2.5515463917525772e-06,
+      "loss": 1.3793,
+      "step": 115
+    },
+    {
+      "epoch": 0.11989664082687339,
+      "grad_norm": 1.5144399404525757,
+      "learning_rate": 2.577319587628866e-06,
+      "loss": 1.2933,
+      "step": 116
+    },
+    {
+      "epoch": 0.12093023255813953,
+      "grad_norm": 1.8202881813049316,
+      "learning_rate": 2.603092783505155e-06,
+      "loss": 1.3358,
+      "step": 117
+    },
+    {
+      "epoch": 0.12196382428940568,
+      "grad_norm": 1.6713426113128662,
+      "learning_rate": 2.628865979381444e-06,
+      "loss": 1.3282,
+      "step": 118
+    },
+    {
+      "epoch": 0.12299741602067184,
+      "grad_norm": 1.6758774518966675,
+      "learning_rate": 2.654639175257732e-06,
+      "loss": 1.3103,
+      "step": 119
+    },
+    {
+      "epoch": 0.12403100775193798,
+      "grad_norm": 1.8854079246520996,
+      "learning_rate": 2.680412371134021e-06,
+      "loss": 1.3777,
+      "step": 120
+    },
+    {
+      "epoch": 0.12506459948320414,
+      "grad_norm": 1.1594630479812622,
+      "learning_rate": 2.7061855670103095e-06,
+      "loss": 1.3589,
+      "step": 121
+    },
+    {
+      "epoch": 0.12609819121447027,
+      "grad_norm": 1.8433988094329834,
+      "learning_rate": 2.731958762886598e-06,
+      "loss": 1.3359,
+      "step": 122
+    },
+    {
+      "epoch": 0.12713178294573643,
+      "grad_norm": 1.655373454093933,
+      "learning_rate": 2.757731958762887e-06,
+      "loss": 1.3365,
+      "step": 123
+    },
+    {
+      "epoch": 0.1281653746770026,
+      "grad_norm": 1.3723413944244385,
+      "learning_rate": 2.7835051546391757e-06,
+      "loss": 1.3367,
+      "step": 124
+    },
+    {
+      "epoch": 0.12919896640826872,
+      "grad_norm": 1.4864723682403564,
+      "learning_rate": 2.809278350515464e-06,
+      "loss": 1.3313,
+      "step": 125
+    },
+    {
+      "epoch": 0.13023255813953488,
+      "grad_norm": 1.9312248229980469,
+      "learning_rate": 2.8350515463917527e-06,
+      "loss": 1.3337,
+      "step": 126
+    },
+    {
+      "epoch": 0.13126614987080104,
+      "grad_norm": 1.6836023330688477,
+      "learning_rate": 2.8608247422680414e-06,
+      "loss": 1.3325,
+      "step": 127
+    },
+    {
+      "epoch": 0.13229974160206717,
+      "grad_norm": 1.4429219961166382,
+      "learning_rate": 2.8865979381443297e-06,
+      "loss": 1.3339,
+      "step": 128
+    },
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 1.8328365087509155,
+      "learning_rate": 2.912371134020619e-06,
+      "loss": 1.3535,
+      "step": 129
+    },
+    {
+      "epoch": 0.1343669250645995,
+      "grad_norm": 1.6869714260101318,
+      "learning_rate": 2.9381443298969076e-06,
+      "loss": 1.3563,
+      "step": 130
+    },
+    {
+      "epoch": 0.13540051679586562,
+      "grad_norm": 1.8647464513778687,
+      "learning_rate": 2.9639175257731963e-06,
+      "loss": 1.3431,
+      "step": 131
+    },
+    {
+      "epoch": 0.13643410852713178,
+      "grad_norm": 1.4633586406707764,
+      "learning_rate": 2.9896907216494846e-06,
+      "loss": 1.2775,
+      "step": 132
+    },
+    {
+      "epoch": 0.13746770025839794,
+      "grad_norm": 1.5387791395187378,
+      "learning_rate": 3.0154639175257733e-06,
+      "loss": 1.3375,
+      "step": 133
+    },
+    {
+      "epoch": 0.13850129198966407,
+      "grad_norm": 1.6977742910385132,
+      "learning_rate": 3.041237113402062e-06,
+      "loss": 1.2724,
+      "step": 134
+    },
+    {
+      "epoch": 0.13953488372093023,
+      "grad_norm": 1.8981420993804932,
+      "learning_rate": 3.067010309278351e-06,
+      "loss": 1.3518,
+      "step": 135
+    },
+    {
+      "epoch": 0.1405684754521964,
+      "grad_norm": 1.464303731918335,
+      "learning_rate": 3.0927835051546395e-06,
+      "loss": 1.3253,
+      "step": 136
+    },
+    {
+      "epoch": 0.14160206718346252,
+      "grad_norm": 1.4378321170806885,
+      "learning_rate": 3.118556701030928e-06,
+      "loss": 1.3194,
+      "step": 137
+    },
+    {
+      "epoch": 0.14263565891472868,
+      "grad_norm": 1.4051882028579712,
+      "learning_rate": 3.1443298969072165e-06,
+      "loss": 1.3438,
+      "step": 138
+    },
+    {
+      "epoch": 0.14366925064599484,
+      "grad_norm": 1.4659231901168823,
+      "learning_rate": 3.170103092783505e-06,
+      "loss": 1.2855,
+      "step": 139
+    },
+    {
+      "epoch": 0.14470284237726097,
+      "grad_norm": 1.6398965120315552,
+      "learning_rate": 3.195876288659794e-06,
+      "loss": 1.2988,
+      "step": 140
+    },
+    {
+      "epoch": 0.14573643410852713,
+      "grad_norm": 2.1323349475860596,
+      "learning_rate": 3.221649484536083e-06,
+      "loss": 1.3068,
+      "step": 141
+    },
+    {
+      "epoch": 0.1467700258397933,
+      "grad_norm": 2.074308395385742,
+      "learning_rate": 3.2474226804123714e-06,
+      "loss": 1.3142,
+      "step": 142
+    },
+    {
+      "epoch": 0.14780361757105942,
+      "grad_norm": 1.7479180097579956,
+      "learning_rate": 3.27319587628866e-06,
+      "loss": 1.3224,
+      "step": 143
+    },
+    {
+      "epoch": 0.14883720930232558,
+      "grad_norm": 1.8256514072418213,
+      "learning_rate": 3.298969072164949e-06,
+      "loss": 1.3094,
+      "step": 144
+    },
+    {
+      "epoch": 0.14987080103359174,
+      "grad_norm": 1.4079580307006836,
+      "learning_rate": 3.324742268041237e-06,
+      "loss": 1.3256,
+      "step": 145
+    },
+    {
+      "epoch": 0.15090439276485787,
+      "grad_norm": 1.598493218421936,
+      "learning_rate": 3.350515463917526e-06,
+      "loss": 1.2855,
+      "step": 146
+    },
+    {
+      "epoch": 0.15193798449612403,
+      "grad_norm": 1.4292045831680298,
+      "learning_rate": 3.376288659793815e-06,
+      "loss": 1.3048,
+      "step": 147
+    },
+    {
+      "epoch": 0.1529715762273902,
+      "grad_norm": 1.9614964723587036,
+      "learning_rate": 3.4020618556701037e-06,
+      "loss": 1.3054,
+      "step": 148
+    },
+    {
+      "epoch": 0.15400516795865632,
+      "grad_norm": 1.69972562789917,
+      "learning_rate": 3.427835051546392e-06,
+      "loss": 1.2852,
+      "step": 149
+    },
+    {
+      "epoch": 0.15503875968992248,
+      "grad_norm": 1.5941336154937744,
+      "learning_rate": 3.4536082474226807e-06,
+      "loss": 1.3247,
+      "step": 150
+    },
+    {
+      "epoch": 0.15503875968992248,
+      "eval_loss": 1.3614064455032349,
+      "eval_runtime": 47.8004,
+      "eval_samples_per_second": 20.92,
+      "eval_steps_per_second": 0.669,
+      "step": 150
+    },
+    {
+      "epoch": 0.15607235142118864,
+      "grad_norm": 1.6188396215438843,
+      "learning_rate": 3.4793814432989694e-06,
+      "loss": 1.3306,
+      "step": 151
+    },
+    {
+      "epoch": 0.15710594315245477,
+      "grad_norm": 1.3866245746612549,
+      "learning_rate": 3.5051546391752577e-06,
+      "loss": 1.3161,
+      "step": 152
+    },
+    {
+      "epoch": 0.15813953488372093,
+      "grad_norm": 1.398223876953125,
+      "learning_rate": 3.530927835051547e-06,
+      "loss": 1.3426,
+      "step": 153
+    },
+    {
+      "epoch": 0.1591731266149871,
+      "grad_norm": 1.534954309463501,
+      "learning_rate": 3.5567010309278356e-06,
+      "loss": 1.3304,
+      "step": 154
+    },
+    {
+      "epoch": 0.16020671834625322,
+      "grad_norm": 1.5275781154632568,
+      "learning_rate": 3.582474226804124e-06,
+      "loss": 1.3096,
+      "step": 155
+    },
+    {
+      "epoch": 0.16124031007751938,
+      "grad_norm": 1.5809870958328247,
+      "learning_rate": 3.6082474226804126e-06,
+      "loss": 1.3282,
+      "step": 156
+    },
+    {
+      "epoch": 0.16227390180878554,
+      "grad_norm": 1.591422200202942,
+      "learning_rate": 3.6340206185567013e-06,
+      "loss": 1.3345,
+      "step": 157
+    },
+    {
+      "epoch": 0.16330749354005167,
+      "grad_norm": 1.3441708087921143,
+      "learning_rate": 3.6597938144329896e-06,
+      "loss": 1.3405,
+      "step": 158
+    },
+    {
+      "epoch": 0.16434108527131783,
+      "grad_norm": 1.5876100063323975,
+      "learning_rate": 3.6855670103092787e-06,
+      "loss": 1.3257,
+      "step": 159
+    },
+    {
+      "epoch": 0.165374677002584,
+      "grad_norm": 1.5338858366012573,
+      "learning_rate": 3.7113402061855674e-06,
+      "loss": 1.3119,
+      "step": 160
+    },
+    {
+      "epoch": 0.16640826873385012,
+      "grad_norm": 1.53935968875885,
+      "learning_rate": 3.737113402061856e-06,
+      "loss": 1.2604,
+      "step": 161
+    },
+    {
+      "epoch": 0.16744186046511628,
+      "grad_norm": 1.5869868993759155,
+      "learning_rate": 3.7628865979381445e-06,
+      "loss": 1.3016,
+      "step": 162
+    },
+    {
+      "epoch": 0.16847545219638244,
+      "grad_norm": 1.5249017477035522,
+      "learning_rate": 3.788659793814433e-06,
+      "loss": 1.3451,
+      "step": 163
+    },
+    {
+      "epoch": 0.16950904392764857,
+      "grad_norm": 1.4573460817337036,
+      "learning_rate": 3.814432989690722e-06,
+      "loss": 1.3037,
+      "step": 164
+    },
+    {
+      "epoch": 0.17054263565891473,
+      "grad_norm": 1.6344059705734253,
+      "learning_rate": 3.840206185567011e-06,
+      "loss": 1.2918,
+      "step": 165
+    },
+    {
+      "epoch": 0.17157622739018089,
+      "grad_norm": 1.7674674987792969,
+      "learning_rate": 3.865979381443299e-06,
+      "loss": 1.2699,
+      "step": 166
+    },
+    {
+      "epoch": 0.17260981912144702,
+      "grad_norm": 1.7461512088775635,
+      "learning_rate": 3.891752577319588e-06,
+      "loss": 1.3533,
+      "step": 167
+    },
+    {
+      "epoch": 0.17364341085271318,
+      "grad_norm": 2.285831928253174,
+      "learning_rate": 3.917525773195877e-06,
+      "loss": 1.2838,
+      "step": 168
+    },
+    {
+      "epoch": 0.17467700258397933,
+      "grad_norm": 1.5369038581848145,
+      "learning_rate": 3.9432989690721655e-06,
+      "loss": 1.2687,
+      "step": 169
+    },
+    {
+      "epoch": 0.17571059431524547,
+      "grad_norm": 1.6334905624389648,
+      "learning_rate": 3.969072164948453e-06,
+      "loss": 1.2232,
+      "step": 170
+    },
+    {
+      "epoch": 0.17674418604651163,
+      "grad_norm": 1.7678533792495728,
+      "learning_rate": 3.994845360824743e-06,
+      "loss": 1.3065,
+      "step": 171
+    },
+    {
+      "epoch": 0.17777777777777778,
+      "grad_norm": 1.4618134498596191,
+      "learning_rate": 4.020618556701032e-06,
+      "loss": 1.2914,
+      "step": 172
+    },
+    {
+      "epoch": 0.17881136950904392,
+      "grad_norm": 1.7900431156158447,
+      "learning_rate": 4.04639175257732e-06,
+      "loss": 1.2476,
+      "step": 173
+    },
+    {
+      "epoch": 0.17984496124031008,
+      "grad_norm": 1.708790898323059,
+      "learning_rate": 4.072164948453608e-06,
+      "loss": 1.308,
+      "step": 174
+    },
+    {
+      "epoch": 0.18087855297157623,
+      "grad_norm": 1.3655714988708496,
+      "learning_rate": 4.097938144329897e-06,
+      "loss": 1.2606,
+      "step": 175
+    },
+    {
+      "epoch": 0.18191214470284237,
+      "grad_norm": 1.6430952548980713,
+      "learning_rate": 4.123711340206186e-06,
+      "loss": 1.2688,
+      "step": 176
+    },
+    {
+      "epoch": 0.18294573643410852,
+      "grad_norm": 1.5826959609985352,
+      "learning_rate": 4.149484536082475e-06,
+      "loss": 1.2699,
+      "step": 177
+    },
+    {
+      "epoch": 0.18397932816537468,
+      "grad_norm": 1.387715458869934,
+      "learning_rate": 4.175257731958763e-06,
+      "loss": 1.283,
+      "step": 178
+    },
+    {
+      "epoch": 0.18501291989664082,
+      "grad_norm": 1.6880526542663574,
+      "learning_rate": 4.201030927835052e-06,
+      "loss": 1.261,
+      "step": 179
+    },
+    {
+      "epoch": 0.18604651162790697,
+      "grad_norm": 1.5378423929214478,
+      "learning_rate": 4.2268041237113405e-06,
+      "loss": 1.2676,
+      "step": 180
+    },
+    {
+      "epoch": 0.18708010335917313,
+      "grad_norm": 1.5375232696533203,
+      "learning_rate": 4.252577319587629e-06,
+      "loss": 1.2835,
+      "step": 181
+    },
+    {
+      "epoch": 0.18811369509043926,
+      "grad_norm": 1.546204686164856,
+      "learning_rate": 4.278350515463918e-06,
+      "loss": 1.2545,
+      "step": 182
+    },
+    {
+      "epoch": 0.18914728682170542,
+      "grad_norm": 1.3169488906860352,
+      "learning_rate": 4.304123711340207e-06,
+      "loss": 1.2659,
+      "step": 183
+    },
+    {
+      "epoch": 0.19018087855297158,
+      "grad_norm": 1.4601619243621826,
+      "learning_rate": 4.329896907216495e-06,
+      "loss": 1.3015,
+      "step": 184
+    },
+    {
+      "epoch": 0.19121447028423771,
+      "grad_norm": 1.3422333002090454,
+      "learning_rate": 4.355670103092784e-06,
+      "loss": 1.2648,
+      "step": 185
+    },
+    {
+      "epoch": 0.19224806201550387,
+      "grad_norm": 1.3259650468826294,
+      "learning_rate": 4.381443298969073e-06,
+      "loss": 1.2831,
+      "step": 186
+    },
+    {
+      "epoch": 0.19328165374677003,
+      "grad_norm": 1.3927111625671387,
+      "learning_rate": 4.407216494845361e-06,
+      "loss": 1.2696,
+      "step": 187
+    },
+    {
+      "epoch": 0.19431524547803616,
+      "grad_norm": 1.364329218864441,
+      "learning_rate": 4.4329896907216494e-06,
+      "loss": 1.2598,
+      "step": 188
+    },
+    {
+      "epoch": 0.19534883720930232,
+      "grad_norm": 1.2551934719085693,
+      "learning_rate": 4.458762886597939e-06,
+      "loss": 1.2769,
+      "step": 189
+    },
+    {
+      "epoch": 0.19638242894056848,
+      "grad_norm": 1.4062399864196777,
+      "learning_rate": 4.484536082474228e-06,
+      "loss": 1.2705,
+      "step": 190
+    },
+    {
+      "epoch": 0.1974160206718346,
+      "grad_norm": 1.4035496711730957,
+      "learning_rate": 4.510309278350516e-06,
+      "loss": 1.2825,
+      "step": 191
+    },
+    {
+      "epoch": 0.19844961240310077,
+      "grad_norm": 1.700435757637024,
+      "learning_rate": 4.536082474226804e-06,
+      "loss": 1.2663,
+      "step": 192
+    },
+    {
+      "epoch": 0.19948320413436693,
+      "grad_norm": 1.2739704847335815,
+      "learning_rate": 4.561855670103093e-06,
+      "loss": 1.3095,
+      "step": 193
+    },
+    {
+      "epoch": 0.20051679586563306,
+      "grad_norm": 1.2628998756408691,
+      "learning_rate": 4.587628865979382e-06,
+      "loss": 1.2704,
+      "step": 194
+    },
+    {
+      "epoch": 0.20155038759689922,
+      "grad_norm": 1.533562421798706,
+      "learning_rate": 4.6134020618556705e-06,
+      "loss": 1.2326,
+      "step": 195
+    },
+    {
+      "epoch": 0.20258397932816538,
+      "grad_norm": 1.5663373470306396,
+      "learning_rate": 4.639175257731959e-06,
+      "loss": 1.2809,
+      "step": 196
+    },
+    {
+      "epoch": 0.2036175710594315,
+      "grad_norm": 1.2257280349731445,
+      "learning_rate": 4.664948453608248e-06,
+      "loss": 1.2824,
+      "step": 197
+    },
+    {
+      "epoch": 0.20465116279069767,
+      "grad_norm": 1.496186375617981,
+      "learning_rate": 4.690721649484537e-06,
+      "loss": 1.2628,
+      "step": 198
+    },
+    {
+      "epoch": 0.20568475452196383,
+      "grad_norm": 1.5209540128707886,
+      "learning_rate": 4.716494845360825e-06,
+      "loss": 1.3015,
+      "step": 199
+    },
+    {
+      "epoch": 0.20671834625322996,
+      "grad_norm": 1.2988780736923218,
+      "learning_rate": 4.742268041237113e-06,
+      "loss": 1.252,
+      "step": 200
+    },
+    {
+      "epoch": 0.20671834625322996,
+      "eval_loss": 1.3255563974380493,
+      "eval_runtime": 47.8452,
+      "eval_samples_per_second": 20.901,
+      "eval_steps_per_second": 0.669,
+      "step": 200
+    },
+    {
+      "epoch": 0.20775193798449612,
+      "grad_norm": 1.4737480878829956,
+      "learning_rate": 4.768041237113403e-06,
+      "loss": 1.2308,
+      "step": 201
+    },
+    {
+      "epoch": 0.20878552971576228,
+      "grad_norm": 1.4353716373443604,
+      "learning_rate": 4.7938144329896915e-06,
+      "loss": 1.2578,
+      "step": 202
+    },
+    {
+      "epoch": 0.2098191214470284,
+      "grad_norm": 1.1848849058151245,
+      "learning_rate": 4.81958762886598e-06,
+      "loss": 1.2498,
+      "step": 203
+    },
+    {
+      "epoch": 0.21085271317829457,
+      "grad_norm": 1.3247244358062744,
+      "learning_rate": 4.845360824742268e-06,
+      "loss": 1.2619,
+      "step": 204
+    },
+    {
+      "epoch": 0.21188630490956073,
+      "grad_norm": 1.4854295253753662,
+      "learning_rate": 4.871134020618557e-06,
+      "loss": 1.2536,
+      "step": 205
+    },
+    {
+      "epoch": 0.21291989664082686,
+      "grad_norm": 1.5427430868148804,
+      "learning_rate": 4.8969072164948455e-06,
+      "loss": 1.2727,
+      "step": 206
+    },
+    {
+      "epoch": 0.21395348837209302,
+      "grad_norm": 1.1259032487869263,
+      "learning_rate": 4.922680412371135e-06,
+      "loss": 1.2364,
+      "step": 207
+    },
+    {
+      "epoch": 0.21498708010335918,
+      "grad_norm": 1.374849557876587,
+      "learning_rate": 4.948453608247423e-06,
+      "loss": 1.2621,
+      "step": 208
+    },
+    {
+      "epoch": 0.2160206718346253,
+      "grad_norm": 1.654929518699646,
+      "learning_rate": 4.974226804123712e-06,
+      "loss": 1.1872,
+      "step": 209
+    },
+    {
+      "epoch": 0.21705426356589147,
+      "grad_norm": 1.6211646795272827,
+      "learning_rate": 5e-06,
+      "loss": 1.2585,
+      "step": 210
+    },
+    {
+      "epoch": 0.21808785529715763,
+      "grad_norm": 1.4611238241195679,
+      "learning_rate": 4.9971264367816096e-06,
+      "loss": 1.2399,
+      "step": 211
+    },
+    {
+      "epoch": 0.21912144702842376,
+      "grad_norm": 1.6246379613876343,
+      "learning_rate": 4.994252873563219e-06,
+      "loss": 1.2592,
+      "step": 212
+    },
+    {
+      "epoch": 0.22015503875968992,
+      "grad_norm": 1.5218091011047363,
+      "learning_rate": 4.991379310344828e-06,
+      "loss": 1.2377,
+      "step": 213
+    },
+    {
+      "epoch": 0.22118863049095608,
+      "grad_norm": 1.5748738050460815,
+      "learning_rate": 4.988505747126437e-06,
+      "loss": 1.2072,
+      "step": 214
+    },
+    {
+      "epoch": 0.2222222222222222,
+      "grad_norm": 1.63685142993927,
+      "learning_rate": 4.985632183908046e-06,
+      "loss": 1.3142,
+      "step": 215
+    },
+    {
+      "epoch": 0.22325581395348837,
+      "grad_norm": 1.6139878034591675,
+      "learning_rate": 4.982758620689655e-06,
+      "loss": 1.2459,
+      "step": 216
+    },
+    {
+      "epoch": 0.22428940568475453,
+      "grad_norm": 1.648569941520691,
+      "learning_rate": 4.9798850574712644e-06,
+      "loss": 1.2329,
+      "step": 217
+    },
+    {
+      "epoch": 0.22532299741602066,
+      "grad_norm": 1.466662883758545,
+      "learning_rate": 4.977011494252874e-06,
+      "loss": 1.2421,
+      "step": 218
+    },
+    {
+      "epoch": 0.22635658914728682,
+      "grad_norm": 1.519354224205017,
+      "learning_rate": 4.9741379310344836e-06,
+      "loss": 1.2964,
+      "step": 219
+    },
+    {
+      "epoch": 0.22739018087855298,
+      "grad_norm": 1.6441487073898315,
+      "learning_rate": 4.971264367816092e-06,
+      "loss": 1.2484,
+      "step": 220
+    },
+    {
+      "epoch": 0.2284237726098191,
+      "grad_norm": 1.721964955329895,
+      "learning_rate": 4.968390804597701e-06,
+      "loss": 1.2443,
+      "step": 221
+    },
+    {
+      "epoch": 0.22945736434108527,
+      "grad_norm": 1.6697990894317627,
+      "learning_rate": 4.965517241379311e-06,
+      "loss": 1.2183,
+      "step": 222
+    },
+    {
+      "epoch": 0.23049095607235143,
+      "grad_norm": 1.4707998037338257,
+      "learning_rate": 4.96264367816092e-06,
+      "loss": 1.2766,
+      "step": 223
+    },
+    {
+      "epoch": 0.23152454780361756,
+      "grad_norm": 1.960856556892395,
+      "learning_rate": 4.959770114942529e-06,
+      "loss": 1.2288,
+      "step": 224
+    },
+    {
+      "epoch": 0.23255813953488372,
+      "grad_norm": 1.502487063407898,
+      "learning_rate": 4.9568965517241384e-06,
+      "loss": 1.2393,
+      "step": 225
+    },
+    {
+      "epoch": 0.23359173126614988,
+      "grad_norm": 1.2606749534606934,
+      "learning_rate": 4.9540229885057476e-06,
+      "loss": 1.2697,
+      "step": 226
+    },
+    {
+      "epoch": 0.234625322997416,
+      "grad_norm": 1.9564929008483887,
+      "learning_rate": 4.951149425287357e-06,
+      "loss": 1.272,
+      "step": 227
+    },
+    {
+      "epoch": 0.23565891472868217,
+      "grad_norm": 1.5385324954986572,
+      "learning_rate": 4.948275862068966e-06,
+      "loss": 1.2819,
+      "step": 228
+    },
+    {
+      "epoch": 0.23669250645994833,
+      "grad_norm": 1.234011173248291,
+      "learning_rate": 4.945402298850575e-06,
+      "loss": 1.2597,
+      "step": 229
+    },
+    {
+      "epoch": 0.23772609819121446,
+      "grad_norm": 1.3302147388458252,
+      "learning_rate": 4.942528735632184e-06,
+      "loss": 1.2118,
+      "step": 230
+    },
+    {
+      "epoch": 0.23875968992248062,
+      "grad_norm": 2.138888359069824,
+      "learning_rate": 4.939655172413793e-06,
+      "loss": 1.2479,
+      "step": 231
+    },
+    {
+      "epoch": 0.23979328165374678,
+      "grad_norm": 1.4027204513549805,
+      "learning_rate": 4.936781609195403e-06,
+      "loss": 1.2577,
+      "step": 232
+    },
+    {
+      "epoch": 0.2408268733850129,
+      "grad_norm": 1.064552664756775,
+      "learning_rate": 4.933908045977012e-06,
+      "loss": 1.1917,
+      "step": 233
+    },
+    {
+      "epoch": 0.24186046511627907,
+      "grad_norm": 1.5456916093826294,
+      "learning_rate": 4.931034482758621e-06,
+      "loss": 1.2093,
+      "step": 234
+    },
+    {
+      "epoch": 0.24289405684754523,
+      "grad_norm": 1.4663442373275757,
+      "learning_rate": 4.92816091954023e-06,
+      "loss": 1.2396,
+      "step": 235
+    },
+    {
+      "epoch": 0.24392764857881136,
+      "grad_norm": 1.4049909114837646,
+      "learning_rate": 4.92528735632184e-06,
+      "loss": 1.1883,
+      "step": 236
+    },
+    {
+      "epoch": 0.24496124031007752,
+      "grad_norm": 1.4005986452102661,
+      "learning_rate": 4.922413793103449e-06,
+      "loss": 1.2572,
+      "step": 237
+    },
+    {
+      "epoch": 0.24599483204134368,
+      "grad_norm": 1.2783763408660889,
+      "learning_rate": 4.919540229885058e-06,
+      "loss": 1.2346,
+      "step": 238
+    },
+    {
+      "epoch": 0.2470284237726098,
+      "grad_norm": 1.2429497241973877,
+      "learning_rate": 4.9166666666666665e-06,
+      "loss": 1.212,
+      "step": 239
+    },
+    {
+      "epoch": 0.24806201550387597,
+      "grad_norm": 1.2838530540466309,
+      "learning_rate": 4.9137931034482765e-06,
+      "loss": 1.2769,
+      "step": 240
+    },
+    {
+      "epoch": 0.24909560723514212,
+      "grad_norm": 1.1848785877227783,
+      "learning_rate": 4.910919540229886e-06,
+      "loss": 1.2526,
+      "step": 241
+    },
+    {
+      "epoch": 0.2501291989664083,
+      "grad_norm": 1.5733468532562256,
+      "learning_rate": 4.908045977011495e-06,
+      "loss": 1.2647,
+      "step": 242
+    },
+    {
+      "epoch": 0.25116279069767444,
+      "grad_norm": 1.34541916847229,
+      "learning_rate": 4.905172413793104e-06,
+      "loss": 1.2421,
+      "step": 243
+    },
+    {
+      "epoch": 0.25219638242894055,
+      "grad_norm": 1.6279263496398926,
+      "learning_rate": 4.902298850574713e-06,
+      "loss": 1.2265,
+      "step": 244
+    },
+    {
+      "epoch": 0.2532299741602067,
+      "grad_norm": 1.5125291347503662,
+      "learning_rate": 4.899425287356322e-06,
+      "loss": 1.2141,
+      "step": 245
+    },
+    {
+      "epoch": 0.25426356589147286,
+      "grad_norm": 1.299894094467163,
+      "learning_rate": 4.896551724137931e-06,
+      "loss": 1.2097,
+      "step": 246
+    },
+    {
+      "epoch": 0.255297157622739,
+      "grad_norm": 1.3731844425201416,
+      "learning_rate": 4.8936781609195405e-06,
+      "loss": 1.2145,
+      "step": 247
+    },
+    {
+      "epoch": 0.2563307493540052,
+      "grad_norm": 1.651253581047058,
+      "learning_rate": 4.89080459770115e-06,
+      "loss": 1.1912,
+      "step": 248
+    },
+    {
+      "epoch": 0.25736434108527134,
+      "grad_norm": 1.1746879816055298,
+      "learning_rate": 4.887931034482759e-06,
+      "loss": 1.2435,
+      "step": 249
+    },
+    {
+      "epoch": 0.25839793281653745,
+      "grad_norm": 1.2900166511535645,
+      "learning_rate": 4.885057471264369e-06,
+      "loss": 1.2003,
+      "step": 250
+    },
+    {
+      "epoch": 0.25839793281653745,
+      "eval_loss": 1.301725149154663,
+      "eval_runtime": 48.0635,
+      "eval_samples_per_second": 20.806,
+      "eval_steps_per_second": 0.666,
+      "step": 250
+    },
+    {
+      "epoch": 0.2594315245478036,
+      "grad_norm": 1.1423178911209106,
+      "learning_rate": 4.882183908045978e-06,
+      "loss": 1.1974,
+      "step": 251
+    },
+    {
+      "epoch": 0.26046511627906976,
+      "grad_norm": 1.1432229280471802,
+      "learning_rate": 4.879310344827586e-06,
+      "loss": 1.2064,
+      "step": 252
+    },
+    {
+      "epoch": 0.2614987080103359,
+      "grad_norm": 1.3829426765441895,
+      "learning_rate": 4.876436781609195e-06,
+      "loss": 1.2864,
+      "step": 253
+    },
+    {
+      "epoch": 0.2625322997416021,
+      "grad_norm": 1.2096635103225708,
+      "learning_rate": 4.873563218390805e-06,
+      "loss": 1.1898,
+      "step": 254
+    },
+    {
+      "epoch": 0.26356589147286824,
+      "grad_norm": 1.4689451456069946,
+      "learning_rate": 4.8706896551724145e-06,
+      "loss": 1.2606,
+      "step": 255
+    },
+    {
+      "epoch": 0.26459948320413434,
+      "grad_norm": 1.1531693935394287,
+      "learning_rate": 4.867816091954024e-06,
+      "loss": 1.2303,
+      "step": 256
+    },
+    {
+      "epoch": 0.2656330749354005,
+      "grad_norm": 1.7083535194396973,
+      "learning_rate": 4.864942528735633e-06,
+      "loss": 1.2353,
+      "step": 257
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.3419526815414429,
+      "learning_rate": 4.862068965517242e-06,
+      "loss": 1.2303,
+      "step": 258
+    },
+    {
+      "epoch": 0.2677002583979328,
+      "grad_norm": 1.5559792518615723,
+      "learning_rate": 4.859195402298851e-06,
+      "loss": 1.2542,
+      "step": 259
+    },
+    {
+      "epoch": 0.268733850129199,
+      "grad_norm": 1.5590232610702515,
+      "learning_rate": 4.85632183908046e-06,
+      "loss": 1.2268,
+      "step": 260
+    },
+    {
+      "epoch": 0.26976744186046514,
+      "grad_norm": 1.3497662544250488,
+      "learning_rate": 4.853448275862069e-06,
+      "loss": 1.2335,
+      "step": 261
+    },
+    {
+      "epoch": 0.27080103359173124,
+      "grad_norm": 1.4309333562850952,
+      "learning_rate": 4.8505747126436785e-06,
+      "loss": 1.2248,
+      "step": 262
+    },
+    {
+      "epoch": 0.2718346253229974,
+      "grad_norm": 1.1154677867889404,
+      "learning_rate": 4.847701149425288e-06,
+      "loss": 1.2418,
+      "step": 263
+    },
+    {
+      "epoch": 0.27286821705426356,
+      "grad_norm": 1.1973319053649902,
+      "learning_rate": 4.844827586206897e-06,
+      "loss": 1.2194,
+      "step": 264
+    },
+    {
+      "epoch": 0.2739018087855297,
+      "grad_norm": 1.5104105472564697,
+      "learning_rate": 4.841954022988506e-06,
+      "loss": 1.2697,
+      "step": 265
+    },
+    {
+      "epoch": 0.2749354005167959,
+      "grad_norm": 1.4947834014892578,
+      "learning_rate": 4.839080459770115e-06,
+      "loss": 1.2369,
+      "step": 266
+    },
+    {
+      "epoch": 0.27596899224806204,
+      "grad_norm": 1.555356502532959,
+      "learning_rate": 4.836206896551724e-06,
+      "loss": 1.2403,
+      "step": 267
+    },
+    {
+      "epoch": 0.27700258397932814,
+      "grad_norm": 1.5942518711090088,
+      "learning_rate": 4.833333333333333e-06,
+      "loss": 1.2013,
+      "step": 268
+    },
+    {
+      "epoch": 0.2780361757105943,
+      "grad_norm": 1.271295189857483,
+      "learning_rate": 4.830459770114943e-06,
+      "loss": 1.2265,
+      "step": 269
+    },
+    {
+      "epoch": 0.27906976744186046,
+      "grad_norm": 1.4804733991622925,
+      "learning_rate": 4.8275862068965525e-06,
+      "loss": 1.2424,
+      "step": 270
+    },
+    {
+      "epoch": 0.2801033591731266,
+      "grad_norm": 1.6949542760849,
+      "learning_rate": 4.824712643678161e-06,
+      "loss": 1.2162,
+      "step": 271
+    },
+    {
+      "epoch": 0.2811369509043928,
+      "grad_norm": 1.4015085697174072,
+      "learning_rate": 4.82183908045977e-06,
+      "loss": 1.2009,
+      "step": 272
+    },
+    {
+      "epoch": 0.28217054263565894,
+      "grad_norm": 1.3916239738464355,
+      "learning_rate": 4.81896551724138e-06,
+      "loss": 1.2221,
+      "step": 273
+    },
+    {
+      "epoch": 0.28320413436692504,
+      "grad_norm": 1.4600123167037964,
+      "learning_rate": 4.816091954022989e-06,
+      "loss": 1.238,
+      "step": 274
+    },
+    {
+      "epoch": 0.2842377260981912,
+      "grad_norm": 1.1622064113616943,
+      "learning_rate": 4.813218390804598e-06,
+      "loss": 1.2032,
+      "step": 275
+    },
+    {
+      "epoch": 0.28527131782945736,
+      "grad_norm": 1.3175055980682373,
+      "learning_rate": 4.810344827586207e-06,
+      "loss": 1.2241,
+      "step": 276
+    },
+    {
+      "epoch": 0.2863049095607235,
+      "grad_norm": 1.3803074359893799,
+      "learning_rate": 4.8074712643678165e-06,
+      "loss": 1.2347,
+      "step": 277
+    },
+    {
+      "epoch": 0.2873385012919897,
+      "grad_norm": 1.5483407974243164,
+      "learning_rate": 4.804597701149426e-06,
+      "loss": 1.2314,
+      "step": 278
+    },
+    {
+      "epoch": 0.28837209302325584,
+      "grad_norm": 1.2761492729187012,
+      "learning_rate": 4.801724137931035e-06,
+      "loss": 1.2209,
+      "step": 279
+    },
+    {
+      "epoch": 0.28940568475452194,
+      "grad_norm": 1.266837477684021,
+      "learning_rate": 4.798850574712644e-06,
+      "loss": 1.2235,
+      "step": 280
+    },
+    {
+      "epoch": 0.2904392764857881,
+      "grad_norm": 1.2987629175186157,
+      "learning_rate": 4.795977011494253e-06,
+      "loss": 1.2239,
+      "step": 281
+    },
+    {
+      "epoch": 0.29147286821705426,
+      "grad_norm": 1.4991391897201538,
+      "learning_rate": 4.793103448275862e-06,
+      "loss": 1.2396,
+      "step": 282
+    },
+    {
+      "epoch": 0.2925064599483204,
+      "grad_norm": 1.325391173362732,
+      "learning_rate": 4.790229885057472e-06,
+      "loss": 1.2246,
+      "step": 283
+    },
+    {
+      "epoch": 0.2935400516795866,
+      "grad_norm": 1.5597681999206543,
+      "learning_rate": 4.7873563218390805e-06,
+      "loss": 1.1917,
+      "step": 284
+    },
+    {
+      "epoch": 0.29457364341085274,
+      "grad_norm": 1.1909379959106445,
+      "learning_rate": 4.78448275862069e-06,
+      "loss": 1.2133,
+      "step": 285
+    },
+    {
+      "epoch": 0.29560723514211884,
+      "grad_norm": 1.2589330673217773,
+      "learning_rate": 4.781609195402299e-06,
+      "loss": 1.2324,
+      "step": 286
+    },
+    {
+      "epoch": 0.296640826873385,
+      "grad_norm": 1.5300198793411255,
+      "learning_rate": 4.778735632183909e-06,
+      "loss": 1.215,
+      "step": 287
+    },
+    {
+      "epoch": 0.29767441860465116,
+      "grad_norm": 1.285587191581726,
+      "learning_rate": 4.775862068965518e-06,
+      "loss": 1.2817,
+      "step": 288
+    },
+    {
+      "epoch": 0.2987080103359173,
+      "grad_norm": 1.579711675643921,
+      "learning_rate": 4.772988505747127e-06,
+      "loss": 1.206,
+      "step": 289
+    },
+    {
+      "epoch": 0.2997416020671835,
+      "grad_norm": 1.2438337802886963,
+      "learning_rate": 4.770114942528735e-06,
+      "loss": 1.2162,
+      "step": 290
+    },
+    {
+      "epoch": 0.30077519379844964,
+      "grad_norm": 1.2078229188919067,
+      "learning_rate": 4.767241379310345e-06,
+      "loss": 1.2262,
+      "step": 291
+    },
+    {
+      "epoch": 0.30180878552971574,
+      "grad_norm": 1.4489924907684326,
+      "learning_rate": 4.7643678160919545e-06,
+      "loss": 1.2028,
+      "step": 292
+    },
+    {
+      "epoch": 0.3028423772609819,
+      "grad_norm": 1.722036600112915,
+      "learning_rate": 4.761494252873564e-06,
+      "loss": 1.2321,
+      "step": 293
+    },
+    {
+      "epoch": 0.30387596899224806,
+      "grad_norm": 1.2497223615646362,
+      "learning_rate": 4.758620689655173e-06,
+      "loss": 1.199,
+      "step": 294
+    },
+    {
+      "epoch": 0.3049095607235142,
+      "grad_norm": 1.1889766454696655,
+      "learning_rate": 4.755747126436782e-06,
+      "loss": 1.2183,
+      "step": 295
+    },
+    {
+      "epoch": 0.3059431524547804,
+      "grad_norm": 1.65548837184906,
+      "learning_rate": 4.752873563218391e-06,
+      "loss": 1.2535,
+      "step": 296
+    },
+    {
+      "epoch": 0.30697674418604654,
+      "grad_norm": 1.7176488637924194,
+      "learning_rate": 4.75e-06,
+      "loss": 1.1354,
+      "step": 297
+    },
+    {
+      "epoch": 0.30801033591731264,
+      "grad_norm": 1.38619863986969,
+      "learning_rate": 4.747126436781609e-06,
+      "loss": 1.2093,
+      "step": 298
+    },
+    {
+      "epoch": 0.3090439276485788,
+      "grad_norm": 1.469096302986145,
+      "learning_rate": 4.7442528735632186e-06,
+      "loss": 1.2218,
+      "step": 299
+    },
+    {
+      "epoch": 0.31007751937984496,
+      "grad_norm": 1.6391884088516235,
+      "learning_rate": 4.741379310344828e-06,
+      "loss": 1.2108,
+      "step": 300
+    },
+    {
+      "epoch": 0.31007751937984496,
+      "eval_loss": 1.2854892015457153,
+      "eval_runtime": 48.1735,
+      "eval_samples_per_second": 20.758,
+      "eval_steps_per_second": 0.664,
+      "step": 300
+    },
+    {
+      "epoch": 0.3111111111111111,
+      "grad_norm": 1.3288958072662354,
+      "learning_rate": 4.738505747126438e-06,
+      "loss": 1.2059,
+      "step": 301
+    },
+    {
+      "epoch": 0.3121447028423773,
+      "grad_norm": 1.5343891382217407,
+      "learning_rate": 4.735632183908047e-06,
+      "loss": 1.2082,
+      "step": 302
+    },
+    {
+      "epoch": 0.31317829457364343,
+      "grad_norm": 1.6383378505706787,
+      "learning_rate": 4.732758620689655e-06,
+      "loss": 1.2331,
+      "step": 303
+    },
+    {
+      "epoch": 0.31421188630490954,
+      "grad_norm": 1.3862324953079224,
+      "learning_rate": 4.729885057471264e-06,
+      "loss": 1.2412,
+      "step": 304
+    },
+    {
+      "epoch": 0.3152454780361757,
+      "grad_norm": 1.4603716135025024,
+      "learning_rate": 4.727011494252874e-06,
+      "loss": 1.1927,
+      "step": 305
+    },
+    {
+      "epoch": 0.31627906976744186,
+      "grad_norm": 1.2502843141555786,
+      "learning_rate": 4.724137931034483e-06,
+      "loss": 1.2226,
+      "step": 306
+    },
+    {
+      "epoch": 0.317312661498708,
+      "grad_norm": 1.6000148057937622,
+      "learning_rate": 4.7212643678160926e-06,
+      "loss": 1.231,
+      "step": 307
+    },
+    {
+      "epoch": 0.3183462532299742,
+      "grad_norm": 1.4646550416946411,
+      "learning_rate": 4.718390804597702e-06,
+      "loss": 1.2653,
+      "step": 308
+    },
+    {
+      "epoch": 0.31937984496124033,
+      "grad_norm": 1.3548325300216675,
+      "learning_rate": 4.715517241379311e-06,
+      "loss": 1.236,
+      "step": 309
+    },
+    {
+      "epoch": 0.32041343669250644,
+      "grad_norm": 1.4769961833953857,
+      "learning_rate": 4.71264367816092e-06,
+      "loss": 1.209,
+      "step": 310
+    },
+    {
+      "epoch": 0.3214470284237726,
+      "grad_norm": 1.2307313680648804,
+      "learning_rate": 4.709770114942529e-06,
+      "loss": 1.206,
+      "step": 311
+    },
+    {
+      "epoch": 0.32248062015503876,
+      "grad_norm": 1.5612189769744873,
+      "learning_rate": 4.706896551724138e-06,
+      "loss": 1.2272,
+      "step": 312
+    },
+    {
+      "epoch": 0.3235142118863049,
+      "grad_norm": 0.9792500734329224,
+      "learning_rate": 4.7040229885057474e-06,
+      "loss": 1.2404,
+      "step": 313
+    },
+    {
+      "epoch": 0.3245478036175711,
+      "grad_norm": 1.2356388568878174,
+      "learning_rate": 4.7011494252873566e-06,
+      "loss": 1.2453,
+      "step": 314
+    },
+    {
+      "epoch": 0.32558139534883723,
+      "grad_norm": 1.3284534215927124,
+      "learning_rate": 4.698275862068966e-06,
+      "loss": 1.2451,
+      "step": 315
+    },
+    {
+      "epoch": 0.32661498708010334,
+      "grad_norm": 1.2313426733016968,
+      "learning_rate": 4.695402298850575e-06,
+      "loss": 1.2073,
+      "step": 316
+    },
+    {
+      "epoch": 0.3276485788113695,
+      "grad_norm": 1.2018225193023682,
+      "learning_rate": 4.692528735632184e-06,
+      "loss": 1.2499,
+      "step": 317
+    },
+    {
+      "epoch": 0.32868217054263565,
+      "grad_norm": 1.0967109203338623,
+      "learning_rate": 4.689655172413793e-06,
+      "loss": 1.1986,
+      "step": 318
+    },
+    {
+      "epoch": 0.3297157622739018,
+      "grad_norm": 1.2187355756759644,
+      "learning_rate": 4.686781609195402e-06,
+      "loss": 1.2364,
+      "step": 319
+    },
+    {
+      "epoch": 0.330749354005168,
+      "grad_norm": 1.3187403678894043,
+      "learning_rate": 4.683908045977012e-06,
+      "loss": 1.1911,
+      "step": 320
+    },
+    {
+      "epoch": 0.33178294573643413,
+      "grad_norm": 1.2736074924468994,
+      "learning_rate": 4.6810344827586214e-06,
+      "loss": 1.1957,
+      "step": 321
+    },
+    {
+      "epoch": 0.33281653746770024,
+      "grad_norm": 1.3698259592056274,
+      "learning_rate": 4.67816091954023e-06,
+      "loss": 1.1875,
+      "step": 322
+    },
+    {
+      "epoch": 0.3338501291989664,
+      "grad_norm": 1.4535542726516724,
+      "learning_rate": 4.675287356321839e-06,
+      "loss": 1.18,
+      "step": 323
+    },
+    {
+      "epoch": 0.33488372093023255,
+      "grad_norm": 1.2355557680130005,
+      "learning_rate": 4.672413793103449e-06,
+      "loss": 1.237,
+      "step": 324
+    },
+    {
+      "epoch": 0.3359173126614987,
+      "grad_norm": 1.1696377992630005,
+      "learning_rate": 4.669540229885058e-06,
+      "loss": 1.2044,
+      "step": 325
+    },
+    {
+      "epoch": 0.33695090439276487,
+      "grad_norm": 1.768639087677002,
+      "learning_rate": 4.666666666666667e-06,
+      "loss": 1.2423,
+      "step": 326
+    },
+    {
+      "epoch": 0.33798449612403103,
+      "grad_norm": 1.7557653188705444,
+      "learning_rate": 4.663793103448276e-06,
+      "loss": 1.1876,
+      "step": 327
+    },
+    {
+      "epoch": 0.33901808785529713,
+      "grad_norm": 1.2822763919830322,
+      "learning_rate": 4.6609195402298855e-06,
+      "loss": 1.1353,
+      "step": 328
+    },
+    {
+      "epoch": 0.3400516795865633,
+      "grad_norm": 1.4557548761367798,
+      "learning_rate": 4.658045977011495e-06,
+      "loss": 1.1894,
+      "step": 329
+    },
+    {
+      "epoch": 0.34108527131782945,
+      "grad_norm": 1.5415141582489014,
+      "learning_rate": 4.655172413793104e-06,
+      "loss": 1.1799,
+      "step": 330
+    },
+    {
+      "epoch": 0.3421188630490956,
+      "grad_norm": 1.247830867767334,
+      "learning_rate": 4.652298850574713e-06,
+      "loss": 1.2636,
+      "step": 331
+    },
+    {
+      "epoch": 0.34315245478036177,
+      "grad_norm": 1.9219553470611572,
+      "learning_rate": 4.649425287356322e-06,
+      "loss": 1.2055,
+      "step": 332
+    },
+    {
+      "epoch": 0.34418604651162793,
+      "grad_norm": 1.318398356437683,
+      "learning_rate": 4.646551724137931e-06,
+      "loss": 1.2062,
+      "step": 333
+    },
+    {
+      "epoch": 0.34521963824289403,
+      "grad_norm": 1.304222583770752,
+      "learning_rate": 4.643678160919541e-06,
+      "loss": 1.1955,
+      "step": 334
+    },
+    {
+      "epoch": 0.3462532299741602,
+      "grad_norm": 1.1557509899139404,
+      "learning_rate": 4.6408045977011495e-06,
+      "loss": 1.2426,
+      "step": 335
+    },
+    {
+      "epoch": 0.34728682170542635,
+      "grad_norm": 1.3262536525726318,
+      "learning_rate": 4.637931034482759e-06,
+      "loss": 1.1738,
+      "step": 336
+    },
+    {
+      "epoch": 0.3483204134366925,
+      "grad_norm": 1.2595785856246948,
+      "learning_rate": 4.635057471264368e-06,
+      "loss": 1.2302,
+      "step": 337
+    },
+    {
+      "epoch": 0.34935400516795867,
+      "grad_norm": 1.6142576932907104,
+      "learning_rate": 4.632183908045978e-06,
+      "loss": 1.2723,
+      "step": 338
+    },
+    {
+      "epoch": 0.35038759689922483,
+      "grad_norm": 1.139347791671753,
+      "learning_rate": 4.629310344827587e-06,
+      "loss": 1.2137,
+      "step": 339
+    },
+    {
+      "epoch": 0.35142118863049093,
+      "grad_norm": 1.1879801750183105,
+      "learning_rate": 4.626436781609196e-06,
+      "loss": 1.2047,
+      "step": 340
+    },
+    {
+      "epoch": 0.3524547803617571,
+      "grad_norm": 1.2845611572265625,
+      "learning_rate": 4.623563218390805e-06,
+      "loss": 1.2227,
+      "step": 341
+    },
+    {
+      "epoch": 0.35348837209302325,
+      "grad_norm": 1.5612064599990845,
+      "learning_rate": 4.620689655172414e-06,
+      "loss": 1.2102,
+      "step": 342
+    },
+    {
+      "epoch": 0.3545219638242894,
+      "grad_norm": 1.1952980756759644,
+      "learning_rate": 4.6178160919540235e-06,
+      "loss": 1.2232,
+      "step": 343
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "grad_norm": 1.310408592224121,
+      "learning_rate": 4.614942528735633e-06,
+      "loss": 1.2293,
+      "step": 344
+    },
+    {
+      "epoch": 0.35658914728682173,
+      "grad_norm": 1.197373390197754,
+      "learning_rate": 4.612068965517242e-06,
+      "loss": 1.2348,
+      "step": 345
+    },
+    {
+      "epoch": 0.35762273901808783,
+      "grad_norm": 1.1051040887832642,
+      "learning_rate": 4.609195402298851e-06,
+      "loss": 1.1952,
+      "step": 346
+    },
+    {
+      "epoch": 0.358656330749354,
+      "grad_norm": 1.2444984912872314,
+      "learning_rate": 4.60632183908046e-06,
+      "loss": 1.1804,
+      "step": 347
+    },
+    {
+      "epoch": 0.35968992248062015,
+      "grad_norm": 1.3992429971694946,
+      "learning_rate": 4.603448275862069e-06,
+      "loss": 1.2365,
+      "step": 348
+    },
+    {
+      "epoch": 0.3607235142118863,
+      "grad_norm": 1.09051513671875,
+      "learning_rate": 4.600574712643678e-06,
+      "loss": 1.1629,
+      "step": 349
+    },
+    {
+      "epoch": 0.36175710594315247,
+      "grad_norm": 1.7428574562072754,
+      "learning_rate": 4.5977011494252875e-06,
+      "loss": 1.2076,
+      "step": 350
+    },
+    {
+      "epoch": 0.36175710594315247,
+      "eval_loss": 1.2728006839752197,
+      "eval_runtime": 48.2021,
+      "eval_samples_per_second": 20.746,
+      "eval_steps_per_second": 0.664,
+      "step": 350
+    },
+    {
+      "epoch": 0.3627906976744186,
+      "grad_norm": 1.1538360118865967,
+      "learning_rate": 4.594827586206897e-06,
+      "loss": 1.1889,
+      "step": 351
+    },
+    {
+      "epoch": 0.36382428940568473,
+      "grad_norm": 1.1787687540054321,
+      "learning_rate": 4.591954022988507e-06,
+      "loss": 1.2185,
+      "step": 352
+    },
+    {
+      "epoch": 0.3648578811369509,
+      "grad_norm": 1.3935431241989136,
+      "learning_rate": 4.589080459770116e-06,
+      "loss": 1.2258,
+      "step": 353
+    },
+    {
+      "epoch": 0.36589147286821705,
+      "grad_norm": 1.6494699716567993,
+      "learning_rate": 4.586206896551724e-06,
+      "loss": 1.1766,
+      "step": 354
+    },
+    {
+      "epoch": 0.3669250645994832,
+      "grad_norm": 1.6146515607833862,
+      "learning_rate": 4.583333333333333e-06,
+      "loss": 1.2243,
+      "step": 355
+    },
+    {
+      "epoch": 0.36795865633074937,
+      "grad_norm": 1.7873674631118774,
+      "learning_rate": 4.580459770114943e-06,
+      "loss": 1.2206,
+      "step": 356
+    },
+    {
+      "epoch": 0.3689922480620155,
+      "grad_norm": 1.2191059589385986,
+      "learning_rate": 4.577586206896552e-06,
+      "loss": 1.205,
+      "step": 357
+    },
+    {
+      "epoch": 0.37002583979328163,
+      "grad_norm": 1.1856673955917358,
+      "learning_rate": 4.5747126436781615e-06,
+      "loss": 1.1971,
+      "step": 358
+    },
+    {
+      "epoch": 0.3710594315245478,
+      "grad_norm": 1.3208264112472534,
+      "learning_rate": 4.571839080459771e-06,
+      "loss": 1.1928,
+      "step": 359
+    },
+    {
+      "epoch": 0.37209302325581395,
+      "grad_norm": 1.5640398263931274,
+      "learning_rate": 4.56896551724138e-06,
+      "loss": 1.2044,
+      "step": 360
+    },
+    {
+      "epoch": 0.3731266149870801,
+      "grad_norm": 1.401240587234497,
+      "learning_rate": 4.566091954022989e-06,
+      "loss": 1.2301,
+      "step": 361
+    },
+    {
+      "epoch": 0.37416020671834627,
+      "grad_norm": 1.1681466102600098,
+      "learning_rate": 4.563218390804598e-06,
+      "loss": 1.1905,
+      "step": 362
+    },
+    {
+      "epoch": 0.3751937984496124,
+      "grad_norm": 1.317150592803955,
+      "learning_rate": 4.560344827586207e-06,
+      "loss": 1.1847,
+      "step": 363
+    },
+    {
+      "epoch": 0.37622739018087853,
+      "grad_norm": 1.352041244506836,
+      "learning_rate": 4.557471264367816e-06,
+      "loss": 1.2131,
+      "step": 364
+    },
+    {
+      "epoch": 0.3772609819121447,
+      "grad_norm": 1.1713517904281616,
+      "learning_rate": 4.5545977011494255e-06,
+      "loss": 1.2225,
+      "step": 365
+    },
+    {
+      "epoch": 0.37829457364341085,
+      "grad_norm": 1.0403046607971191,
+      "learning_rate": 4.551724137931035e-06,
+      "loss": 1.2035,
+      "step": 366
+    },
+    {
+      "epoch": 0.379328165374677,
+      "grad_norm": 1.5299835205078125,
+      "learning_rate": 4.548850574712644e-06,
+      "loss": 1.2136,
+      "step": 367
+    },
+    {
+      "epoch": 0.38036175710594317,
+      "grad_norm": 1.5781452655792236,
+      "learning_rate": 4.545977011494253e-06,
+      "loss": 1.1985,
+      "step": 368
+    },
+    {
+      "epoch": 0.3813953488372093,
+      "grad_norm": 1.2083759307861328,
+      "learning_rate": 4.543103448275862e-06,
+      "loss": 1.2026,
+      "step": 369
+    },
+    {
+      "epoch": 0.38242894056847543,
+      "grad_norm": 1.3300983905792236,
+      "learning_rate": 4.540229885057471e-06,
+      "loss": 1.1836,
+      "step": 370
+    },
+    {
+      "epoch": 0.3834625322997416,
+      "grad_norm": 1.4870195388793945,
+      "learning_rate": 4.537356321839081e-06,
+      "loss": 1.2231,
+      "step": 371
+    },
+    {
+      "epoch": 0.38449612403100775,
+      "grad_norm": 1.2063935995101929,
+      "learning_rate": 4.53448275862069e-06,
+      "loss": 1.1906,
+      "step": 372
+    },
+    {
+      "epoch": 0.3855297157622739,
+      "grad_norm": 1.3956559896469116,
+      "learning_rate": 4.5316091954022995e-06,
+      "loss": 1.2301,
+      "step": 373
+    },
+    {
+      "epoch": 0.38656330749354006,
+      "grad_norm": 1.1331489086151123,
+      "learning_rate": 4.528735632183908e-06,
+      "loss": 1.2028,
+      "step": 374
+    },
+    {
+      "epoch": 0.3875968992248062,
+      "grad_norm": 1.4904316663742065,
+      "learning_rate": 4.525862068965518e-06,
+      "loss": 1.1915,
+      "step": 375
+    },
+    {
+      "epoch": 0.3886304909560723,
+      "grad_norm": 1.1676017045974731,
+      "learning_rate": 4.522988505747127e-06,
+      "loss": 1.1624,
+      "step": 376
+    },
+    {
+      "epoch": 0.3896640826873385,
+      "grad_norm": 1.2875901460647583,
+      "learning_rate": 4.520114942528736e-06,
+      "loss": 1.2227,
+      "step": 377
+    },
+    {
+      "epoch": 0.39069767441860465,
+      "grad_norm": 1.2845635414123535,
+      "learning_rate": 4.517241379310345e-06,
+      "loss": 1.2218,
+      "step": 378
+    },
+    {
+      "epoch": 0.3917312661498708,
+      "grad_norm": 1.3359668254852295,
+      "learning_rate": 4.514367816091954e-06,
+      "loss": 1.1604,
+      "step": 379
+    },
+    {
+      "epoch": 0.39276485788113696,
+      "grad_norm": 1.3478506803512573,
+      "learning_rate": 4.5114942528735635e-06,
+      "loss": 1.1677,
+      "step": 380
+    },
+    {
+      "epoch": 0.3937984496124031,
+      "grad_norm": 1.0987980365753174,
+      "learning_rate": 4.508620689655173e-06,
+      "loss": 1.1661,
+      "step": 381
+    },
+    {
+      "epoch": 0.3948320413436692,
+      "grad_norm": 1.3515042066574097,
+      "learning_rate": 4.505747126436782e-06,
+      "loss": 1.2192,
+      "step": 382
+    },
+    {
+      "epoch": 0.3958656330749354,
+      "grad_norm": 1.5531221628189087,
+      "learning_rate": 4.502873563218391e-06,
+      "loss": 1.1888,
+      "step": 383
+    },
+    {
+      "epoch": 0.39689922480620154,
+      "grad_norm": 1.011456847190857,
+      "learning_rate": 4.5e-06,
+      "loss": 1.1671,
+      "step": 384
+    },
+    {
+      "epoch": 0.3979328165374677,
+      "grad_norm": 1.3113720417022705,
+      "learning_rate": 4.49712643678161e-06,
+      "loss": 1.1745,
+      "step": 385
+    },
+    {
+      "epoch": 0.39896640826873386,
+      "grad_norm": 1.6751645803451538,
+      "learning_rate": 4.494252873563218e-06,
+      "loss": 1.2111,
+      "step": 386
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.4498629570007324,
+      "learning_rate": 4.4913793103448275e-06,
+      "loss": 1.2099,
+      "step": 387
+    },
+    {
+      "epoch": 0.4010335917312661,
+      "grad_norm": 1.3328680992126465,
+      "learning_rate": 4.488505747126437e-06,
+      "loss": 1.1965,
+      "step": 388
+    },
+    {
+      "epoch": 0.4020671834625323,
+      "grad_norm": 1.3834558725357056,
+      "learning_rate": 4.485632183908047e-06,
+      "loss": 1.2279,
+      "step": 389
+    },
+    {
+      "epoch": 0.40310077519379844,
+      "grad_norm": 1.228224754333496,
+      "learning_rate": 4.482758620689656e-06,
+      "loss": 1.1387,
+      "step": 390
+    },
+    {
+      "epoch": 0.4041343669250646,
+      "grad_norm": 1.141613245010376,
+      "learning_rate": 4.479885057471265e-06,
+      "loss": 1.2034,
+      "step": 391
+    },
+    {
+      "epoch": 0.40516795865633076,
+      "grad_norm": 1.2839511632919312,
+      "learning_rate": 4.477011494252874e-06,
+      "loss": 1.1951,
+      "step": 392
+    },
+    {
+      "epoch": 0.4062015503875969,
+      "grad_norm": 1.5047776699066162,
+      "learning_rate": 4.474137931034483e-06,
+      "loss": 1.1889,
+      "step": 393
+    },
+    {
+      "epoch": 0.407235142118863,
+      "grad_norm": 1.5040572881698608,
+      "learning_rate": 4.471264367816092e-06,
+      "loss": 1.1624,
+      "step": 394
+    },
+    {
+      "epoch": 0.4082687338501292,
+      "grad_norm": 1.1555923223495483,
+      "learning_rate": 4.4683908045977016e-06,
+      "loss": 1.2081,
+      "step": 395
+    },
+    {
+      "epoch": 0.40930232558139534,
+      "grad_norm": 1.3135583400726318,
+      "learning_rate": 4.465517241379311e-06,
+      "loss": 1.1992,
+      "step": 396
+    },
+    {
+      "epoch": 0.4103359173126615,
+      "grad_norm": 1.0716359615325928,
+      "learning_rate": 4.46264367816092e-06,
+      "loss": 1.1869,
+      "step": 397
+    },
+    {
+      "epoch": 0.41136950904392766,
+      "grad_norm": 1.249202847480774,
+      "learning_rate": 4.459770114942529e-06,
+      "loss": 1.1742,
+      "step": 398
+    },
+    {
+      "epoch": 0.4124031007751938,
+      "grad_norm": 1.2279281616210938,
+      "learning_rate": 4.456896551724138e-06,
+      "loss": 1.1665,
+      "step": 399
+    },
+    {
+      "epoch": 0.4134366925064599,
+      "grad_norm": 0.9841963052749634,
+      "learning_rate": 4.454022988505747e-06,
+      "loss": 1.2191,
+      "step": 400
+    },
+    {
+      "epoch": 0.4134366925064599,
+      "eval_loss": 1.2680693864822388,
+      "eval_runtime": 48.4023,
+      "eval_samples_per_second": 20.66,
+      "eval_steps_per_second": 0.661,
+      "step": 400
+    },
+    {
+      "epoch": 0.4144702842377261,
+      "grad_norm": 1.42137610912323,
+      "learning_rate": 4.4511494252873564e-06,
+      "loss": 1.1756,
+      "step": 401
+    },
+    {
+      "epoch": 0.41550387596899224,
+      "grad_norm": 1.0953741073608398,
+      "learning_rate": 4.4482758620689656e-06,
+      "loss": 1.219,
+      "step": 402
+    },
+    {
+      "epoch": 0.4165374677002584,
+      "grad_norm": 1.101485013961792,
+      "learning_rate": 4.4454022988505756e-06,
+      "loss": 1.1576,
+      "step": 403
+    },
+    {
+      "epoch": 0.41757105943152456,
+      "grad_norm": 1.3133827447891235,
+      "learning_rate": 4.442528735632185e-06,
+      "loss": 1.16,
+      "step": 404
+    },
+    {
+      "epoch": 0.4186046511627907,
+      "grad_norm": 1.1481531858444214,
+      "learning_rate": 4.439655172413794e-06,
+      "loss": 1.1493,
+      "step": 405
+    },
+    {
+      "epoch": 0.4196382428940568,
+      "grad_norm": 1.3434548377990723,
+      "learning_rate": 4.436781609195402e-06,
+      "loss": 1.1506,
+      "step": 406
+    },
+    {
+      "epoch": 0.420671834625323,
+      "grad_norm": 1.102965235710144,
+      "learning_rate": 4.433908045977012e-06,
+      "loss": 1.1446,
+      "step": 407
+    },
+    {
+      "epoch": 0.42170542635658914,
+      "grad_norm": 1.3014981746673584,
+      "learning_rate": 4.431034482758621e-06,
+      "loss": 1.1921,
+      "step": 408
+    },
+    {
+      "epoch": 0.4227390180878553,
+      "grad_norm": 1.3584136962890625,
+      "learning_rate": 4.4281609195402304e-06,
+      "loss": 1.2074,
+      "step": 409
+    },
+    {
+      "epoch": 0.42377260981912146,
+      "grad_norm": 1.6955718994140625,
+      "learning_rate": 4.42528735632184e-06,
+      "loss": 1.2293,
+      "step": 410
+    },
+    {
+      "epoch": 0.4248062015503876,
+      "grad_norm": 1.273550033569336,
+      "learning_rate": 4.422413793103449e-06,
+      "loss": 1.1775,
+      "step": 411
+    },
+    {
+      "epoch": 0.4258397932816537,
+      "grad_norm": 1.1694968938827515,
+      "learning_rate": 4.419540229885058e-06,
+      "loss": 1.1528,
+      "step": 412
+    },
+    {
+      "epoch": 0.4268733850129199,
+      "grad_norm": 1.134385108947754,
+      "learning_rate": 4.416666666666667e-06,
+      "loss": 1.2074,
+      "step": 413
+    },
+    {
+      "epoch": 0.42790697674418604,
+      "grad_norm": 1.3740071058273315,
+      "learning_rate": 4.413793103448276e-06,
+      "loss": 1.1792,
+      "step": 414
+    },
+    {
+      "epoch": 0.4289405684754522,
+      "grad_norm": 1.0105552673339844,
+      "learning_rate": 4.410919540229885e-06,
+      "loss": 1.198,
+      "step": 415
+    },
+    {
+      "epoch": 0.42997416020671836,
+      "grad_norm": 1.2491987943649292,
+      "learning_rate": 4.4080459770114944e-06,
+      "loss": 1.2049,
+      "step": 416
+    },
+    {
+      "epoch": 0.4310077519379845,
+      "grad_norm": 1.2663400173187256,
+      "learning_rate": 4.405172413793104e-06,
+      "loss": 1.2276,
+      "step": 417
+    },
+    {
+      "epoch": 0.4320413436692506,
+      "grad_norm": 1.1918857097625732,
+      "learning_rate": 4.402298850574713e-06,
+      "loss": 1.1922,
+      "step": 418
+    },
+    {
+      "epoch": 0.4330749354005168,
+      "grad_norm": 1.4146064519882202,
+      "learning_rate": 4.399425287356322e-06,
+      "loss": 1.2117,
+      "step": 419
+    },
+    {
+      "epoch": 0.43410852713178294,
+      "grad_norm": 1.4689112901687622,
+      "learning_rate": 4.396551724137931e-06,
+      "loss": 1.1674,
+      "step": 420
+    },
+    {
+      "epoch": 0.4351421188630491,
+      "grad_norm": 1.2137223482131958,
+      "learning_rate": 4.39367816091954e-06,
+      "loss": 1.2254,
+      "step": 421
+    },
+    {
+      "epoch": 0.43617571059431526,
+      "grad_norm": 1.5300006866455078,
+      "learning_rate": 4.39080459770115e-06,
+      "loss": 1.2286,
+      "step": 422
+    },
+    {
+      "epoch": 0.4372093023255814,
+      "grad_norm": 1.0909467935562134,
+      "learning_rate": 4.387931034482759e-06,
+      "loss": 1.2015,
+      "step": 423
+    },
+    {
+      "epoch": 0.4382428940568475,
+      "grad_norm": 1.1437002420425415,
+      "learning_rate": 4.3850574712643685e-06,
+      "loss": 1.1486,
+      "step": 424
+    },
+    {
+      "epoch": 0.4392764857881137,
+      "grad_norm": 1.2198647260665894,
+      "learning_rate": 4.382183908045977e-06,
+      "loss": 1.1961,
+      "step": 425
+    },
+    {
+      "epoch": 0.44031007751937984,
+      "grad_norm": 1.4140639305114746,
+      "learning_rate": 4.379310344827587e-06,
+      "loss": 1.186,
+      "step": 426
+    },
+    {
+      "epoch": 0.441343669250646,
+      "grad_norm": 1.1367213726043701,
+      "learning_rate": 4.376436781609196e-06,
+      "loss": 1.1975,
+      "step": 427
+    },
+    {
+      "epoch": 0.44237726098191216,
+      "grad_norm": 1.3699185848236084,
+      "learning_rate": 4.373563218390805e-06,
+      "loss": 1.2206,
+      "step": 428
+    },
+    {
+      "epoch": 0.4434108527131783,
+      "grad_norm": 1.152096152305603,
+      "learning_rate": 4.370689655172414e-06,
+      "loss": 1.1758,
+      "step": 429
+    },
+    {
+      "epoch": 0.4444444444444444,
+      "grad_norm": 1.2542738914489746,
+      "learning_rate": 4.367816091954023e-06,
+      "loss": 1.1657,
+      "step": 430
+    },
+    {
+      "epoch": 0.4454780361757106,
+      "grad_norm": 1.3231966495513916,
+      "learning_rate": 4.3649425287356325e-06,
+      "loss": 1.1864,
+      "step": 431
+    },
+    {
+      "epoch": 0.44651162790697674,
+      "grad_norm": 1.4225741624832153,
+      "learning_rate": 4.362068965517242e-06,
+      "loss": 1.1983,
+      "step": 432
+    },
+    {
+      "epoch": 0.4475452196382429,
+      "grad_norm": 1.2822364568710327,
+      "learning_rate": 4.359195402298851e-06,
+      "loss": 1.1802,
+      "step": 433
+    },
+    {
+      "epoch": 0.44857881136950906,
+      "grad_norm": 1.3587886095046997,
+      "learning_rate": 4.35632183908046e-06,
+      "loss": 1.1926,
+      "step": 434
+    },
+    {
+      "epoch": 0.4496124031007752,
+      "grad_norm": 1.5335510969161987,
+      "learning_rate": 4.353448275862069e-06,
+      "loss": 1.1817,
+      "step": 435
+    },
+    {
+      "epoch": 0.4506459948320413,
+      "grad_norm": 1.1684523820877075,
+      "learning_rate": 4.350574712643679e-06,
+      "loss": 1.1831,
+      "step": 436
+    },
+    {
+      "epoch": 0.4516795865633075,
+      "grad_norm": 1.1984566450119019,
+      "learning_rate": 4.347701149425288e-06,
+      "loss": 1.1373,
+      "step": 437
+    },
+    {
+      "epoch": 0.45271317829457364,
+      "grad_norm": 1.2617523670196533,
+      "learning_rate": 4.3448275862068965e-06,
+      "loss": 1.1713,
+      "step": 438
+    },
+    {
+      "epoch": 0.4537467700258398,
+      "grad_norm": 1.2004725933074951,
+      "learning_rate": 4.341954022988506e-06,
+      "loss": 1.1768,
+      "step": 439
+    },
+    {
+      "epoch": 0.45478036175710596,
+      "grad_norm": 1.4468493461608887,
+      "learning_rate": 4.339080459770116e-06,
+      "loss": 1.1597,
+      "step": 440
+    },
+    {
+      "epoch": 0.4558139534883721,
+      "grad_norm": 1.1114490032196045,
+      "learning_rate": 4.336206896551725e-06,
+      "loss": 1.1791,
+      "step": 441
+    },
+    {
+      "epoch": 0.4568475452196382,
+      "grad_norm": 1.325484037399292,
+      "learning_rate": 4.333333333333334e-06,
+      "loss": 1.1689,
+      "step": 442
+    },
+    {
+      "epoch": 0.4578811369509044,
+      "grad_norm": 1.3673852682113647,
+      "learning_rate": 4.330459770114943e-06,
+      "loss": 1.173,
+      "step": 443
+    },
+    {
+      "epoch": 0.45891472868217054,
+      "grad_norm": 1.173619031906128,
+      "learning_rate": 4.327586206896552e-06,
+      "loss": 1.1969,
+      "step": 444
+    },
+    {
+      "epoch": 0.4599483204134367,
+      "grad_norm": 1.1025069952011108,
+      "learning_rate": 4.324712643678161e-06,
+      "loss": 1.1858,
+      "step": 445
+    },
+    {
+      "epoch": 0.46098191214470285,
+      "grad_norm": 1.2002084255218506,
+      "learning_rate": 4.3218390804597705e-06,
+      "loss": 1.1835,
+      "step": 446
+    },
+    {
+      "epoch": 0.462015503875969,
+      "grad_norm": 1.1930514574050903,
+      "learning_rate": 4.31896551724138e-06,
+      "loss": 1.1871,
+      "step": 447
+    },
+    {
+      "epoch": 0.4630490956072351,
+      "grad_norm": 1.4340951442718506,
+      "learning_rate": 4.316091954022989e-06,
+      "loss": 1.1525,
+      "step": 448
+    },
+    {
+      "epoch": 0.4640826873385013,
+      "grad_norm": 1.5040807723999023,
+      "learning_rate": 4.313218390804598e-06,
+      "loss": 1.18,
+      "step": 449
+    },
+    {
+      "epoch": 0.46511627906976744,
+      "grad_norm": 1.2207226753234863,
+      "learning_rate": 4.310344827586207e-06,
+      "loss": 1.177,
+      "step": 450
+    },
+    {
+      "epoch": 0.46511627906976744,
+      "eval_loss": 1.2576266527175903,
+      "eval_runtime": 48.4102,
+      "eval_samples_per_second": 20.657,
+      "eval_steps_per_second": 0.661,
+      "step": 450
+    },
+    {
+      "epoch": 0.4661498708010336,
+      "grad_norm": 1.4215892553329468,
+      "learning_rate": 4.307471264367816e-06,
+      "loss": 1.1994,
+      "step": 451
+    },
+    {
+      "epoch": 0.46718346253229975,
+      "grad_norm": 1.152346134185791,
+      "learning_rate": 4.304597701149425e-06,
+      "loss": 1.1397,
+      "step": 452
+    },
+    {
+      "epoch": 0.4682170542635659,
+      "grad_norm": 1.2140494585037231,
+      "learning_rate": 4.3017241379310345e-06,
+      "loss": 1.1908,
+      "step": 453
+    },
+    {
+      "epoch": 0.469250645994832,
+      "grad_norm": 1.2626186609268188,
+      "learning_rate": 4.2988505747126445e-06,
+      "loss": 1.1904,
+      "step": 454
+    },
+    {
+      "epoch": 0.4702842377260982,
+      "grad_norm": 1.5459721088409424,
+      "learning_rate": 4.295977011494254e-06,
+      "loss": 1.1698,
+      "step": 455
+    },
+    {
+      "epoch": 0.47131782945736433,
+      "grad_norm": 1.2738758325576782,
+      "learning_rate": 4.293103448275863e-06,
+      "loss": 1.1824,
+      "step": 456
+    },
+    {
+      "epoch": 0.4723514211886305,
+      "grad_norm": 1.1245863437652588,
+      "learning_rate": 4.290229885057471e-06,
+      "loss": 1.1698,
+      "step": 457
+    },
+    {
+      "epoch": 0.47338501291989665,
+      "grad_norm": 1.2073432207107544,
+      "learning_rate": 4.287356321839081e-06,
+      "loss": 1.132,
+      "step": 458
+    },
+    {
+      "epoch": 0.4744186046511628,
+      "grad_norm": 1.552643060684204,
+      "learning_rate": 4.28448275862069e-06,
+      "loss": 1.2027,
+      "step": 459
+    },
+    {
+      "epoch": 0.4754521963824289,
+      "grad_norm": 1.4971299171447754,
+      "learning_rate": 4.281609195402299e-06,
+      "loss": 1.1758,
+      "step": 460
+    },
+    {
+      "epoch": 0.4764857881136951,
+      "grad_norm": 1.0017420053482056,
+      "learning_rate": 4.2787356321839085e-06,
+      "loss": 1.1464,
+      "step": 461
+    },
+    {
+      "epoch": 0.47751937984496123,
+      "grad_norm": 1.210776925086975,
+      "learning_rate": 4.275862068965518e-06,
+      "loss": 1.1984,
+      "step": 462
+    },
+    {
+      "epoch": 0.4785529715762274,
+      "grad_norm": 1.0808087587356567,
+      "learning_rate": 4.272988505747127e-06,
+      "loss": 1.2179,
+      "step": 463
+    },
+    {
+      "epoch": 0.47958656330749355,
+      "grad_norm": 1.408263921737671,
+      "learning_rate": 4.270114942528736e-06,
+      "loss": 1.1446,
+      "step": 464
+    },
+    {
+      "epoch": 0.4806201550387597,
+      "grad_norm": 1.3446722030639648,
+      "learning_rate": 4.267241379310345e-06,
+      "loss": 1.1894,
+      "step": 465
+    },
+    {
+      "epoch": 0.4816537467700258,
+      "grad_norm": 1.1083803176879883,
+      "learning_rate": 4.264367816091954e-06,
+      "loss": 1.1972,
+      "step": 466
+    },
+    {
+      "epoch": 0.482687338501292,
+      "grad_norm": 1.3985694646835327,
+      "learning_rate": 4.261494252873563e-06,
+      "loss": 1.2348,
+      "step": 467
+    },
+    {
+      "epoch": 0.48372093023255813,
+      "grad_norm": 1.1992748975753784,
+      "learning_rate": 4.2586206896551725e-06,
+      "loss": 1.1702,
+      "step": 468
+    },
+    {
+      "epoch": 0.4847545219638243,
+      "grad_norm": 1.3031333684921265,
+      "learning_rate": 4.2557471264367825e-06,
+      "loss": 1.1977,
+      "step": 469
+    },
+    {
+      "epoch": 0.48578811369509045,
+      "grad_norm": 1.161007285118103,
+      "learning_rate": 4.252873563218391e-06,
+      "loss": 1.1841,
+      "step": 470
+    },
+    {
+      "epoch": 0.4868217054263566,
+      "grad_norm": 1.0703585147857666,
+      "learning_rate": 4.25e-06,
+      "loss": 1.1654,
+      "step": 471
+    },
+    {
+      "epoch": 0.4878552971576227,
+      "grad_norm": 1.2260897159576416,
+      "learning_rate": 4.247126436781609e-06,
+      "loss": 1.1918,
+      "step": 472
+    },
+    {
+      "epoch": 0.4888888888888889,
+      "grad_norm": 1.16552734375,
+      "learning_rate": 4.244252873563219e-06,
+      "loss": 1.1821,
+      "step": 473
+    },
+    {
+      "epoch": 0.48992248062015503,
+      "grad_norm": 1.2570924758911133,
+      "learning_rate": 4.241379310344828e-06,
+      "loss": 1.1526,
+      "step": 474
+    },
+    {
+      "epoch": 0.4909560723514212,
+      "grad_norm": 1.1815022230148315,
+      "learning_rate": 4.238505747126437e-06,
+      "loss": 1.1735,
+      "step": 475
+    },
+    {
+      "epoch": 0.49198966408268735,
+      "grad_norm": 1.4533460140228271,
+      "learning_rate": 4.235632183908046e-06,
+      "loss": 1.1875,
+      "step": 476
+    },
+    {
+      "epoch": 0.4930232558139535,
+      "grad_norm": 1.09726881980896,
+      "learning_rate": 4.232758620689656e-06,
+      "loss": 1.1541,
+      "step": 477
+    },
+    {
+      "epoch": 0.4940568475452196,
+      "grad_norm": 1.1364957094192505,
+      "learning_rate": 4.229885057471265e-06,
+      "loss": 1.1704,
+      "step": 478
+    },
+    {
+      "epoch": 0.49509043927648577,
+      "grad_norm": 1.0956162214279175,
+      "learning_rate": 4.227011494252874e-06,
+      "loss": 1.159,
+      "step": 479
+    },
+    {
+      "epoch": 0.49612403100775193,
+      "grad_norm": 1.3513314723968506,
+      "learning_rate": 4.224137931034483e-06,
+      "loss": 1.1436,
+      "step": 480
+    },
+    {
+      "epoch": 0.4971576227390181,
+      "grad_norm": 1.3028982877731323,
+      "learning_rate": 4.221264367816092e-06,
+      "loss": 1.1976,
+      "step": 481
+    },
+    {
+      "epoch": 0.49819121447028425,
+      "grad_norm": 1.1886882781982422,
+      "learning_rate": 4.218390804597701e-06,
+      "loss": 1.1843,
+      "step": 482
+    },
+    {
+      "epoch": 0.4992248062015504,
+      "grad_norm": 1.1816167831420898,
+      "learning_rate": 4.2155172413793106e-06,
+      "loss": 1.1305,
+      "step": 483
+    },
+    {
+      "epoch": 0.5002583979328166,
+      "grad_norm": 1.1279879808425903,
+      "learning_rate": 4.21264367816092e-06,
+      "loss": 1.1278,
+      "step": 484
+    },
+    {
+      "epoch": 0.5012919896640827,
+      "grad_norm": 1.1833527088165283,
+      "learning_rate": 4.209770114942529e-06,
+      "loss": 1.1395,
+      "step": 485
+    },
+    {
+      "epoch": 0.5023255813953489,
+      "grad_norm": 1.0197025537490845,
+      "learning_rate": 4.206896551724138e-06,
+      "loss": 1.1979,
+      "step": 486
+    },
+    {
+      "epoch": 0.5033591731266149,
+      "grad_norm": 1.1991444826126099,
+      "learning_rate": 4.204022988505748e-06,
+      "loss": 1.1873,
+      "step": 487
+    },
+    {
+      "epoch": 0.5043927648578811,
+      "grad_norm": 1.2488261461257935,
+      "learning_rate": 4.201149425287357e-06,
+      "loss": 1.1615,
+      "step": 488
+    },
+    {
+      "epoch": 0.5054263565891473,
+      "grad_norm": 1.042624592781067,
+      "learning_rate": 4.1982758620689654e-06,
+      "loss": 1.1975,
+      "step": 489
+    },
+    {
+      "epoch": 0.5064599483204134,
+      "grad_norm": 1.2511039972305298,
+      "learning_rate": 4.1954022988505746e-06,
+      "loss": 1.1775,
+      "step": 490
+    },
+    {
+      "epoch": 0.5074935400516796,
+      "grad_norm": 1.2317752838134766,
+      "learning_rate": 4.1925287356321846e-06,
+      "loss": 1.2116,
+      "step": 491
+    },
+    {
+      "epoch": 0.5085271317829457,
+      "grad_norm": 1.1231718063354492,
+      "learning_rate": 4.189655172413794e-06,
+      "loss": 1.1769,
+      "step": 492
+    },
+    {
+      "epoch": 0.5095607235142119,
+      "grad_norm": 1.025221347808838,
+      "learning_rate": 4.186781609195403e-06,
+      "loss": 1.1715,
+      "step": 493
+    },
+    {
+      "epoch": 0.510594315245478,
+      "grad_norm": 1.2898812294006348,
+      "learning_rate": 4.183908045977012e-06,
+      "loss": 1.223,
+      "step": 494
+    },
+    {
+      "epoch": 0.5116279069767442,
+      "grad_norm": 1.2475007772445679,
+      "learning_rate": 4.181034482758621e-06,
+      "loss": 1.167,
+      "step": 495
+    },
+    {
+      "epoch": 0.5126614987080104,
+      "grad_norm": 1.4511774778366089,
+      "learning_rate": 4.17816091954023e-06,
+      "loss": 1.1787,
+      "step": 496
+    },
+    {
+      "epoch": 0.5136950904392765,
+      "grad_norm": 1.3853965997695923,
+      "learning_rate": 4.1752873563218394e-06,
+      "loss": 1.1974,
+      "step": 497
+    },
+    {
+      "epoch": 0.5147286821705427,
+      "grad_norm": 1.340599536895752,
+      "learning_rate": 4.1724137931034486e-06,
+      "loss": 1.1505,
+      "step": 498
+    },
+    {
+      "epoch": 0.5157622739018087,
+      "grad_norm": 1.0093344449996948,
+      "learning_rate": 4.169540229885058e-06,
+      "loss": 1.1297,
+      "step": 499
+    },
+    {
+      "epoch": 0.5167958656330749,
+      "grad_norm": 1.3832353353500366,
+      "learning_rate": 4.166666666666667e-06,
+      "loss": 1.174,
+      "step": 500
+    },
+    {
+      "epoch": 0.5167958656330749,
+      "eval_loss": 1.2501999139785767,
+      "eval_runtime": 48.1533,
+      "eval_samples_per_second": 20.767,
+      "eval_steps_per_second": 0.665,
+      "step": 500
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1934,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.87708552822784e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}