Upload folder using huggingface_hub

checkpoint-2603/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/OLMo-2-1124-7B-bho_Deva-coli-mean/",
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100384
+}

checkpoint-2603/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 100257,
+  "pad_token_id": 100277,
+  "transformers_version": "4.48.0.dev0"
+}

checkpoint-2603/model-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15512ca6ec088773cfa40a688ea570ab2238e0ebc048f907fb41664a3ed062eb
+size 4950070472

checkpoint-2603/model-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0b6a357d03676c4986588aaf4dc96afc3f44a0498514961d2b2cfa1940655e
+size 4991622816

checkpoint-2603/model-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1038a7d346a681a8fb6949e054c2246f3f3c80e826855c1cec6af1155cc607d
+size 4924546896

checkpoint-2603/model-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c0a21146ace02d36928787ae7f34be868f7c81b894dea98ec7b0e5a3ea01b2
+size 4857404928

checkpoint-2603/model-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f82a3014ccfc0f67208f63785ae4bc71f09aba86db48bf6139d18178abbfed
+size 4857404928

checkpoint-2603/model-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09d7f35172b78383375af56cfbcae8ad4d4d99b01cdcde3c3a09582b305b9d5b
+size 4614509080

checkpoint-2603/model.safetensors.index.json

@@ -0,0 +1,362 @@
+{
+  "metadata": {
+    "total_size": 29195517952
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

checkpoint-2603/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f15ecbf4778a4a75a97c368c9a04c6ae1a582fe5a110f897da16df40debbea5
+size 13055369358

checkpoint-2603/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-2603/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c10eab28581440371942f16961c25d6c2080c871324aab61f489518f0a454e35
+size 1064

checkpoint-2603/trainer_state.json

@@ -0,0 +1,3064 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 2603,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002305032654629274,
+      "grad_norm": 53.856021881103516,
+      "learning_rate": 2.2988505747126437e-06,
+      "loss": 6.4748,
+      "step": 6
+    },
+    {
+      "epoch": 0.004610065309258548,
+      "grad_norm": 10.265009880065918,
+      "learning_rate": 4.5977011494252875e-06,
+      "loss": 5.3163,
+      "step": 12
+    },
+    {
+      "epoch": 0.006915097963887821,
+      "grad_norm": 7.959039211273193,
+      "learning_rate": 6.896551724137932e-06,
+      "loss": 4.7064,
+      "step": 18
+    },
+    {
+      "epoch": 0.009220130618517095,
+      "grad_norm": 5.8295488357543945,
+      "learning_rate": 9.195402298850575e-06,
+      "loss": 4.3612,
+      "step": 24
+    },
+    {
+      "epoch": 0.01152516327314637,
+      "grad_norm": 4.496989727020264,
+      "learning_rate": 1.1494252873563218e-05,
+      "loss": 4.1643,
+      "step": 30
+    },
+    {
+      "epoch": 0.013830195927775643,
+      "grad_norm": 3.9524970054626465,
+      "learning_rate": 1.3793103448275863e-05,
+      "loss": 3.9844,
+      "step": 36
+    },
+    {
+      "epoch": 0.016135228582404917,
+      "grad_norm": 4.384060382843018,
+      "learning_rate": 1.6091954022988507e-05,
+      "loss": 3.7331,
+      "step": 42
+    },
+    {
+      "epoch": 0.01844026123703419,
+      "grad_norm": 5.873193264007568,
+      "learning_rate": 1.839080459770115e-05,
+      "loss": 3.7501,
+      "step": 48
+    },
+    {
+      "epoch": 0.020745293891663467,
+      "grad_norm": 12.467376708984375,
+      "learning_rate": 2.0689655172413793e-05,
+      "loss": 3.6747,
+      "step": 54
+    },
+    {
+      "epoch": 0.02305032654629274,
+      "grad_norm": 6.967535972595215,
+      "learning_rate": 2.2988505747126437e-05,
+      "loss": 3.5072,
+      "step": 60
+    },
+    {
+      "epoch": 0.025355359200922013,
+      "grad_norm": 8.085759162902832,
+      "learning_rate": 2.5287356321839083e-05,
+      "loss": 3.4384,
+      "step": 66
+    },
+    {
+      "epoch": 0.027660391855551286,
+      "grad_norm": 6.835882663726807,
+      "learning_rate": 2.7586206896551727e-05,
+      "loss": 3.4143,
+      "step": 72
+    },
+    {
+      "epoch": 0.029965424510180562,
+      "grad_norm": 4.951820373535156,
+      "learning_rate": 2.988505747126437e-05,
+      "loss": 3.2636,
+      "step": 78
+    },
+    {
+      "epoch": 0.032270457164809835,
+      "grad_norm": 4.204140663146973,
+      "learning_rate": 3.218390804597701e-05,
+      "loss": 3.1916,
+      "step": 84
+    },
+    {
+      "epoch": 0.03457548981943911,
+      "grad_norm": 3.802454710006714,
+      "learning_rate": 3.4482758620689657e-05,
+      "loss": 3.1395,
+      "step": 90
+    },
+    {
+      "epoch": 0.03688052247406838,
+      "grad_norm": 5.0679850578308105,
+      "learning_rate": 3.67816091954023e-05,
+      "loss": 3.094,
+      "step": 96
+    },
+    {
+      "epoch": 0.039185555128697654,
+      "grad_norm": 3.6308820247650146,
+      "learning_rate": 3.908045977011495e-05,
+      "loss": 3.0388,
+      "step": 102
+    },
+    {
+      "epoch": 0.041490587783326933,
+      "grad_norm": 3.776557683944702,
+      "learning_rate": 4.1379310344827587e-05,
+      "loss": 2.9748,
+      "step": 108
+    },
+    {
+      "epoch": 0.043795620437956206,
+      "grad_norm": 3.4992129802703857,
+      "learning_rate": 4.367816091954024e-05,
+      "loss": 2.9827,
+      "step": 114
+    },
+    {
+      "epoch": 0.04610065309258548,
+      "grad_norm": 3.097571849822998,
+      "learning_rate": 4.597701149425287e-05,
+      "loss": 2.8381,
+      "step": 120
+    },
+    {
+      "epoch": 0.04840568574721475,
+      "grad_norm": 3.4623844623565674,
+      "learning_rate": 4.827586206896552e-05,
+      "loss": 2.7752,
+      "step": 126
+    },
+    {
+      "epoch": 0.050710718401844025,
+      "grad_norm": 3.1929099559783936,
+      "learning_rate": 5.057471264367817e-05,
+      "loss": 2.8002,
+      "step": 132
+    },
+    {
+      "epoch": 0.0530157510564733,
+      "grad_norm": 3.454050064086914,
+      "learning_rate": 5.287356321839081e-05,
+      "loss": 2.724,
+      "step": 138
+    },
+    {
+      "epoch": 0.05532078371110257,
+      "grad_norm": 3.4763906002044678,
+      "learning_rate": 5.517241379310345e-05,
+      "loss": 2.6884,
+      "step": 144
+    },
+    {
+      "epoch": 0.05762581636573185,
+      "grad_norm": 3.204158067703247,
+      "learning_rate": 5.747126436781609e-05,
+      "loss": 2.6505,
+      "step": 150
+    },
+    {
+      "epoch": 0.059930849020361124,
+      "grad_norm": 2.9979729652404785,
+      "learning_rate": 5.977011494252874e-05,
+      "loss": 2.6749,
+      "step": 156
+    },
+    {
+      "epoch": 0.0622358816749904,
+      "grad_norm": 2.607539176940918,
+      "learning_rate": 6.206896551724138e-05,
+      "loss": 2.6104,
+      "step": 162
+    },
+    {
+      "epoch": 0.06454091432961967,
+      "grad_norm": 2.3872625827789307,
+      "learning_rate": 6.436781609195403e-05,
+      "loss": 2.6504,
+      "step": 168
+    },
+    {
+      "epoch": 0.06684594698424895,
+      "grad_norm": 2.667975902557373,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 2.5319,
+      "step": 174
+    },
+    {
+      "epoch": 0.06915097963887822,
+      "grad_norm": 3.090562582015991,
+      "learning_rate": 6.896551724137931e-05,
+      "loss": 2.5326,
+      "step": 180
+    },
+    {
+      "epoch": 0.0714560122935075,
+      "grad_norm": 2.7412374019622803,
+      "learning_rate": 7.126436781609196e-05,
+      "loss": 2.5683,
+      "step": 186
+    },
+    {
+      "epoch": 0.07376104494813676,
+      "grad_norm": 2.471371650695801,
+      "learning_rate": 7.35632183908046e-05,
+      "loss": 2.5179,
+      "step": 192
+    },
+    {
+      "epoch": 0.07606607760276604,
+      "grad_norm": 2.625797986984253,
+      "learning_rate": 7.586206896551724e-05,
+      "loss": 2.5301,
+      "step": 198
+    },
+    {
+      "epoch": 0.07837111025739531,
+      "grad_norm": 3.2228333950042725,
+      "learning_rate": 7.81609195402299e-05,
+      "loss": 2.4392,
+      "step": 204
+    },
+    {
+      "epoch": 0.08067614291202459,
+      "grad_norm": 2.780992031097412,
+      "learning_rate": 8.045977011494253e-05,
+      "loss": 2.4826,
+      "step": 210
+    },
+    {
+      "epoch": 0.08298117556665387,
+      "grad_norm": 3.0570290088653564,
+      "learning_rate": 8.275862068965517e-05,
+      "loss": 2.4565,
+      "step": 216
+    },
+    {
+      "epoch": 0.08528620822128313,
+      "grad_norm": 2.3220102787017822,
+      "learning_rate": 8.505747126436782e-05,
+      "loss": 2.5089,
+      "step": 222
+    },
+    {
+      "epoch": 0.08759124087591241,
+      "grad_norm": 2.4583637714385986,
+      "learning_rate": 8.735632183908047e-05,
+      "loss": 2.3985,
+      "step": 228
+    },
+    {
+      "epoch": 0.08989627353054168,
+      "grad_norm": 2.4556047916412354,
+      "learning_rate": 8.96551724137931e-05,
+      "loss": 2.4429,
+      "step": 234
+    },
+    {
+      "epoch": 0.09220130618517096,
+      "grad_norm": 3.328052282333374,
+      "learning_rate": 9.195402298850575e-05,
+      "loss": 2.414,
+      "step": 240
+    },
+    {
+      "epoch": 0.09450633883980022,
+      "grad_norm": 3.1558191776275635,
+      "learning_rate": 9.425287356321839e-05,
+      "loss": 2.3621,
+      "step": 246
+    },
+    {
+      "epoch": 0.0968113714944295,
+      "grad_norm": 2.6115167140960693,
+      "learning_rate": 9.655172413793105e-05,
+      "loss": 2.4482,
+      "step": 252
+    },
+    {
+      "epoch": 0.09911640414905878,
+      "grad_norm": 2.6599738597869873,
+      "learning_rate": 9.885057471264369e-05,
+      "loss": 2.4507,
+      "step": 258
+    },
+    {
+      "epoch": 0.10142143680368805,
+      "grad_norm": 2.867357015609741,
+      "learning_rate": 9.999990918668272e-05,
+      "loss": 2.4109,
+      "step": 264
+    },
+    {
+      "epoch": 0.10372646945831733,
+      "grad_norm": 2.4468297958374023,
+      "learning_rate": 9.999918268212369e-05,
+      "loss": 2.408,
+      "step": 270
+    },
+    {
+      "epoch": 0.1060315021129466,
+      "grad_norm": 2.1774463653564453,
+      "learning_rate": 9.999772968356182e-05,
+      "loss": 2.3625,
+      "step": 276
+    },
+    {
+      "epoch": 0.10833653476757588,
+      "grad_norm": 2.2998595237731934,
+      "learning_rate": 9.999555021210937e-05,
+      "loss": 2.4588,
+      "step": 282
+    },
+    {
+      "epoch": 0.11064156742220514,
+      "grad_norm": 2.3958585262298584,
+      "learning_rate": 9.999264429943431e-05,
+      "loss": 2.372,
+      "step": 288
+    },
+    {
+      "epoch": 0.11294660007683442,
+      "grad_norm": 2.728062868118286,
+      "learning_rate": 9.99890119877599e-05,
+      "loss": 2.3477,
+      "step": 294
+    },
+    {
+      "epoch": 0.1152516327314637,
+      "grad_norm": 2.2501401901245117,
+      "learning_rate": 9.998465332986401e-05,
+      "loss": 2.2814,
+      "step": 300
+    },
+    {
+      "epoch": 0.11755666538609297,
+      "grad_norm": 2.2734405994415283,
+      "learning_rate": 9.997956838907852e-05,
+      "loss": 2.3092,
+      "step": 306
+    },
+    {
+      "epoch": 0.11986169804072225,
+      "grad_norm": 2.6800060272216797,
+      "learning_rate": 9.997375723928817e-05,
+      "loss": 2.2877,
+      "step": 312
+    },
+    {
+      "epoch": 0.12216673069535151,
+      "grad_norm": 2.206369400024414,
+      "learning_rate": 9.996721996492965e-05,
+      "loss": 2.2894,
+      "step": 318
+    },
+    {
+      "epoch": 0.1244717633499808,
+      "grad_norm": 2.4944238662719727,
+      "learning_rate": 9.995995666099034e-05,
+      "loss": 2.2698,
+      "step": 324
+    },
+    {
+      "epoch": 0.12677679600461006,
+      "grad_norm": 2.422243356704712,
+      "learning_rate": 9.995196743300692e-05,
+      "loss": 2.193,
+      "step": 330
+    },
+    {
+      "epoch": 0.12908182865923934,
+      "grad_norm": 2.5400567054748535,
+      "learning_rate": 9.994325239706377e-05,
+      "loss": 2.163,
+      "step": 336
+    },
+    {
+      "epoch": 0.13138686131386862,
+      "grad_norm": 2.392552375793457,
+      "learning_rate": 9.99338116797914e-05,
+      "loss": 2.2356,
+      "step": 342
+    },
+    {
+      "epoch": 0.1336918939684979,
+      "grad_norm": 2.4796054363250732,
+      "learning_rate": 9.992364541836459e-05,
+      "loss": 2.2747,
+      "step": 348
+    },
+    {
+      "epoch": 0.13599692662312715,
+      "grad_norm": 3.121873378753662,
+      "learning_rate": 9.991275376050024e-05,
+      "loss": 2.1307,
+      "step": 354
+    },
+    {
+      "epoch": 0.13830195927775643,
+      "grad_norm": 3.140073776245117,
+      "learning_rate": 9.990113686445546e-05,
+      "loss": 2.3051,
+      "step": 360
+    },
+    {
+      "epoch": 0.1406069919323857,
+      "grad_norm": 1.9879580736160278,
+      "learning_rate": 9.988879489902509e-05,
+      "loss": 2.338,
+      "step": 366
+    },
+    {
+      "epoch": 0.142912024587015,
+      "grad_norm": 2.337467670440674,
+      "learning_rate": 9.987572804353937e-05,
+      "loss": 2.2738,
+      "step": 372
+    },
+    {
+      "epoch": 0.14521705724164424,
+      "grad_norm": 2.5424106121063232,
+      "learning_rate": 9.986193648786124e-05,
+      "loss": 2.2342,
+      "step": 378
+    },
+    {
+      "epoch": 0.14752208989627352,
+      "grad_norm": 2.880481719970703,
+      "learning_rate": 9.98474204323836e-05,
+      "loss": 2.2469,
+      "step": 384
+    },
+    {
+      "epoch": 0.1498271225509028,
+      "grad_norm": 2.5551915168762207,
+      "learning_rate": 9.983218008802648e-05,
+      "loss": 2.1399,
+      "step": 390
+    },
+    {
+      "epoch": 0.15213215520553208,
+      "grad_norm": 2.556264877319336,
+      "learning_rate": 9.981621567623385e-05,
+      "loss": 2.2329,
+      "step": 396
+    },
+    {
+      "epoch": 0.15443718786016136,
+      "grad_norm": 2.3260979652404785,
+      "learning_rate": 9.979952742897049e-05,
+      "loss": 2.2576,
+      "step": 402
+    },
+    {
+      "epoch": 0.15674222051479061,
+      "grad_norm": 2.241696834564209,
+      "learning_rate": 9.978211558871861e-05,
+      "loss": 2.1743,
+      "step": 408
+    },
+    {
+      "epoch": 0.1590472531694199,
+      "grad_norm": 2.444596767425537,
+      "learning_rate": 9.976398040847428e-05,
+      "loss": 2.1517,
+      "step": 414
+    },
+    {
+      "epoch": 0.16135228582404917,
+      "grad_norm": 2.300065040588379,
+      "learning_rate": 9.974512215174381e-05,
+      "loss": 2.1751,
+      "step": 420
+    },
+    {
+      "epoch": 0.16365731847867845,
+      "grad_norm": 2.6848104000091553,
+      "learning_rate": 9.972554109253989e-05,
+      "loss": 2.2504,
+      "step": 426
+    },
+    {
+      "epoch": 0.16596235113330773,
+      "grad_norm": 2.403059959411621,
+      "learning_rate": 9.97052375153776e-05,
+      "loss": 2.1172,
+      "step": 432
+    },
+    {
+      "epoch": 0.16826738378793699,
+      "grad_norm": 3.4649202823638916,
+      "learning_rate": 9.968421171527032e-05,
+      "loss": 2.1344,
+      "step": 438
+    },
+    {
+      "epoch": 0.17057241644256627,
+      "grad_norm": 1.8871408700942993,
+      "learning_rate": 9.966246399772538e-05,
+      "loss": 2.0402,
+      "step": 444
+    },
+    {
+      "epoch": 0.17287744909719555,
+      "grad_norm": 2.056946039199829,
+      "learning_rate": 9.96399946787397e-05,
+      "loss": 2.093,
+      "step": 450
+    },
+    {
+      "epoch": 0.17518248175182483,
+      "grad_norm": 2.5751354694366455,
+      "learning_rate": 9.961680408479507e-05,
+      "loss": 2.1116,
+      "step": 456
+    },
+    {
+      "epoch": 0.17748751440645408,
+      "grad_norm": 2.126829147338867,
+      "learning_rate": 9.959289255285363e-05,
+      "loss": 2.1042,
+      "step": 462
+    },
+    {
+      "epoch": 0.17979254706108336,
+      "grad_norm": 2.328918933868408,
+      "learning_rate": 9.956826043035268e-05,
+      "loss": 2.1705,
+      "step": 468
+    },
+    {
+      "epoch": 0.18209757971571264,
+      "grad_norm": 2.0855398178100586,
+      "learning_rate": 9.954290807519988e-05,
+      "loss": 1.995,
+      "step": 474
+    },
+    {
+      "epoch": 0.18440261237034192,
+      "grad_norm": 2.647292375564575,
+      "learning_rate": 9.951683585576794e-05,
+      "loss": 2.0674,
+      "step": 480
+    },
+    {
+      "epoch": 0.1867076450249712,
+      "grad_norm": 2.3290131092071533,
+      "learning_rate": 9.949004415088927e-05,
+      "loss": 2.1138,
+      "step": 486
+    },
+    {
+      "epoch": 0.18901267767960045,
+      "grad_norm": 2.2102277278900146,
+      "learning_rate": 9.94625333498505e-05,
+      "loss": 2.042,
+      "step": 492
+    },
+    {
+      "epoch": 0.19131771033422973,
+      "grad_norm": 2.2257633209228516,
+      "learning_rate": 9.943430385238677e-05,
+      "loss": 1.9499,
+      "step": 498
+    },
+    {
+      "epoch": 0.193622742988859,
+      "grad_norm": 2.250659942626953,
+      "learning_rate": 9.940535606867602e-05,
+      "loss": 2.0843,
+      "step": 504
+    },
+    {
+      "epoch": 0.1959277756434883,
+      "grad_norm": 1.6855454444885254,
+      "learning_rate": 9.937569041933294e-05,
+      "loss": 2.019,
+      "step": 510
+    },
+    {
+      "epoch": 0.19823280829811757,
+      "grad_norm": 2.0027201175689697,
+      "learning_rate": 9.934530733540294e-05,
+      "loss": 2.058,
+      "step": 516
+    },
+    {
+      "epoch": 0.20053784095274682,
+      "grad_norm": 1.920457124710083,
+      "learning_rate": 9.931420725835576e-05,
+      "loss": 2.2252,
+      "step": 522
+    },
+    {
+      "epoch": 0.2028428736073761,
+      "grad_norm": 1.9815555810928345,
+      "learning_rate": 9.928239064007918e-05,
+      "loss": 2.0853,
+      "step": 528
+    },
+    {
+      "epoch": 0.20514790626200538,
+      "grad_norm": 2.40604567527771,
+      "learning_rate": 9.924985794287242e-05,
+      "loss": 2.1158,
+      "step": 534
+    },
+    {
+      "epoch": 0.20745293891663466,
+      "grad_norm": 2.5315816402435303,
+      "learning_rate": 9.92166096394394e-05,
+      "loss": 2.016,
+      "step": 540
+    },
+    {
+      "epoch": 0.2097579715712639,
+      "grad_norm": 2.700812339782715,
+      "learning_rate": 9.918264621288187e-05,
+      "loss": 2.029,
+      "step": 546
+    },
+    {
+      "epoch": 0.2120630042258932,
+      "grad_norm": 1.9871690273284912,
+      "learning_rate": 9.91479681566924e-05,
+      "loss": 1.9834,
+      "step": 552
+    },
+    {
+      "epoch": 0.21436803688052247,
+      "grad_norm": 2.945413589477539,
+      "learning_rate": 9.911257597474722e-05,
+      "loss": 2.0259,
+      "step": 558
+    },
+    {
+      "epoch": 0.21667306953515175,
+      "grad_norm": 2.000675916671753,
+      "learning_rate": 9.907647018129894e-05,
+      "loss": 2.2123,
+      "step": 564
+    },
+    {
+      "epoch": 0.21897810218978103,
+      "grad_norm": 2.347562789916992,
+      "learning_rate": 9.903965130096891e-05,
+      "loss": 2.0911,
+      "step": 570
+    },
+    {
+      "epoch": 0.22128313484441028,
+      "grad_norm": 1.9191615581512451,
+      "learning_rate": 9.900211986873985e-05,
+      "loss": 1.92,
+      "step": 576
+    },
+    {
+      "epoch": 0.22358816749903956,
+      "grad_norm": 1.851443886756897,
+      "learning_rate": 9.896387642994787e-05,
+      "loss": 1.9798,
+      "step": 582
+    },
+    {
+      "epoch": 0.22589320015366884,
+      "grad_norm": 1.849892020225525,
+      "learning_rate": 9.892492154027462e-05,
+      "loss": 1.9961,
+      "step": 588
+    },
+    {
+      "epoch": 0.22819823280829812,
+      "grad_norm": 2.072357654571533,
+      "learning_rate": 9.888525576573922e-05,
+      "loss": 1.9663,
+      "step": 594
+    },
+    {
+      "epoch": 0.2305032654629274,
+      "grad_norm": 2.1180946826934814,
+      "learning_rate": 9.884487968269006e-05,
+      "loss": 2.1093,
+      "step": 600
+    },
+    {
+      "epoch": 0.23280829811755666,
+      "grad_norm": 2.36588978767395,
+      "learning_rate": 9.880379387779637e-05,
+      "loss": 2.0478,
+      "step": 606
+    },
+    {
+      "epoch": 0.23511333077218594,
+      "grad_norm": 2.0738325119018555,
+      "learning_rate": 9.87619989480397e-05,
+      "loss": 2.0701,
+      "step": 612
+    },
+    {
+      "epoch": 0.23741836342681522,
+      "grad_norm": 2.005675792694092,
+      "learning_rate": 9.871949550070533e-05,
+      "loss": 2.0751,
+      "step": 618
+    },
+    {
+      "epoch": 0.2397233960814445,
+      "grad_norm": 1.8945198059082031,
+      "learning_rate": 9.867628415337333e-05,
+      "loss": 2.0073,
+      "step": 624
+    },
+    {
+      "epoch": 0.24202842873607375,
+      "grad_norm": 1.743676781654358,
+      "learning_rate": 9.863236553390965e-05,
+      "loss": 2.0169,
+      "step": 630
+    },
+    {
+      "epoch": 0.24433346139070303,
+      "grad_norm": 2.078378200531006,
+      "learning_rate": 9.8587740280457e-05,
+      "loss": 1.9831,
+      "step": 636
+    },
+    {
+      "epoch": 0.2466384940453323,
+      "grad_norm": 1.9130889177322388,
+      "learning_rate": 9.854240904142556e-05,
+      "loss": 2.1173,
+      "step": 642
+    },
+    {
+      "epoch": 0.2489435266999616,
+      "grad_norm": 2.135650634765625,
+      "learning_rate": 9.849637247548356e-05,
+      "loss": 1.9329,
+      "step": 648
+    },
+    {
+      "epoch": 0.25124855935459084,
+      "grad_norm": 2.4106664657592773,
+      "learning_rate": 9.844963125154773e-05,
+      "loss": 2.0074,
+      "step": 654
+    },
+    {
+      "epoch": 0.2535535920092201,
+      "grad_norm": 1.9850528240203857,
+      "learning_rate": 9.840218604877354e-05,
+      "loss": 2.0373,
+      "step": 660
+    },
+    {
+      "epoch": 0.2558586246638494,
+      "grad_norm": 1.9373111724853516,
+      "learning_rate": 9.835403755654536e-05,
+      "loss": 1.9882,
+      "step": 666
+    },
+    {
+      "epoch": 0.2581636573184787,
+      "grad_norm": 2.355316400527954,
+      "learning_rate": 9.830518647446643e-05,
+      "loss": 1.9861,
+      "step": 672
+    },
+    {
+      "epoch": 0.26046868997310796,
+      "grad_norm": 1.7912750244140625,
+      "learning_rate": 9.825563351234874e-05,
+      "loss": 1.9285,
+      "step": 678
+    },
+    {
+      "epoch": 0.26277372262773724,
+      "grad_norm": 1.6330952644348145,
+      "learning_rate": 9.820537939020264e-05,
+      "loss": 2.0204,
+      "step": 684
+    },
+    {
+      "epoch": 0.2650787552823665,
+      "grad_norm": 1.7869762182235718,
+      "learning_rate": 9.815442483822646e-05,
+      "loss": 2.0133,
+      "step": 690
+    },
+    {
+      "epoch": 0.2673837879369958,
+      "grad_norm": 1.738743782043457,
+      "learning_rate": 9.81027705967958e-05,
+      "loss": 1.9677,
+      "step": 696
+    },
+    {
+      "epoch": 0.269688820591625,
+      "grad_norm": 1.8138949871063232,
+      "learning_rate": 9.805041741645287e-05,
+      "loss": 1.8719,
+      "step": 702
+    },
+    {
+      "epoch": 0.2719938532462543,
+      "grad_norm": 1.620474934577942,
+      "learning_rate": 9.799736605789553e-05,
+      "loss": 1.8931,
+      "step": 708
+    },
+    {
+      "epoch": 0.2742988859008836,
+      "grad_norm": 1.9721405506134033,
+      "learning_rate": 9.794361729196629e-05,
+      "loss": 1.9911,
+      "step": 714
+    },
+    {
+      "epoch": 0.27660391855551286,
+      "grad_norm": 1.7554848194122314,
+      "learning_rate": 9.7889171899641e-05,
+      "loss": 2.0013,
+      "step": 720
+    },
+    {
+      "epoch": 0.27890895121014214,
+      "grad_norm": 2.014599561691284,
+      "learning_rate": 9.783403067201762e-05,
+      "loss": 1.9576,
+      "step": 726
+    },
+    {
+      "epoch": 0.2812139838647714,
+      "grad_norm": 1.8215922117233276,
+      "learning_rate": 9.777819441030467e-05,
+      "loss": 2.1209,
+      "step": 732
+    },
+    {
+      "epoch": 0.2835190165194007,
+      "grad_norm": 1.7385495901107788,
+      "learning_rate": 9.772166392580962e-05,
+      "loss": 2.0566,
+      "step": 738
+    },
+    {
+      "epoch": 0.28582404917403,
+      "grad_norm": 2.2076575756073,
+      "learning_rate": 9.766444003992703e-05,
+      "loss": 1.9881,
+      "step": 744
+    },
+    {
+      "epoch": 0.28812908182865926,
+      "grad_norm": 2.0855135917663574,
+      "learning_rate": 9.76065235841267e-05,
+      "loss": 1.9397,
+      "step": 750
+    },
+    {
+      "epoch": 0.2904341144832885,
+      "grad_norm": 2.3066651821136475,
+      "learning_rate": 9.754791539994152e-05,
+      "loss": 1.911,
+      "step": 756
+    },
+    {
+      "epoch": 0.29273914713791777,
+      "grad_norm": 1.8872731924057007,
+      "learning_rate": 9.748861633895534e-05,
+      "loss": 1.965,
+      "step": 762
+    },
+    {
+      "epoch": 0.29504417979254705,
+      "grad_norm": 1.6439388990402222,
+      "learning_rate": 9.742862726279047e-05,
+      "loss": 1.8993,
+      "step": 768
+    },
+    {
+      "epoch": 0.2973492124471763,
+      "grad_norm": 1.7374255657196045,
+      "learning_rate": 9.736794904309523e-05,
+      "loss": 1.7805,
+      "step": 774
+    },
+    {
+      "epoch": 0.2996542451018056,
+      "grad_norm": 1.6874982118606567,
+      "learning_rate": 9.730658256153129e-05,
+      "loss": 1.8397,
+      "step": 780
+    },
+    {
+      "epoch": 0.3019592777564349,
+      "grad_norm": 1.902889370918274,
+      "learning_rate": 9.724452870976085e-05,
+      "loss": 1.9701,
+      "step": 786
+    },
+    {
+      "epoch": 0.30426431041106416,
+      "grad_norm": 1.8667631149291992,
+      "learning_rate": 9.718178838943364e-05,
+      "loss": 1.9539,
+      "step": 792
+    },
+    {
+      "epoch": 0.30656934306569344,
+      "grad_norm": 2.0487167835235596,
+      "learning_rate": 9.711836251217394e-05,
+      "loss": 1.9868,
+      "step": 798
+    },
+    {
+      "epoch": 0.3088743757203227,
+      "grad_norm": 1.7226240634918213,
+      "learning_rate": 9.705425199956716e-05,
+      "loss": 1.9893,
+      "step": 804
+    },
+    {
+      "epoch": 0.311179408374952,
+      "grad_norm": 1.6886826753616333,
+      "learning_rate": 9.698945778314662e-05,
+      "loss": 1.9505,
+      "step": 810
+    },
+    {
+      "epoch": 0.31348444102958123,
+      "grad_norm": 1.9105143547058105,
+      "learning_rate": 9.69239808043799e-05,
+      "loss": 1.9514,
+      "step": 816
+    },
+    {
+      "epoch": 0.3157894736842105,
+      "grad_norm": 1.8108230829238892,
+      "learning_rate": 9.685782201465518e-05,
+      "loss": 1.9346,
+      "step": 822
+    },
+    {
+      "epoch": 0.3180945063388398,
+      "grad_norm": 1.9173780679702759,
+      "learning_rate": 9.679098237526745e-05,
+      "loss": 1.9719,
+      "step": 828
+    },
+    {
+      "epoch": 0.32039953899346907,
+      "grad_norm": 1.75904381275177,
+      "learning_rate": 9.672346285740455e-05,
+      "loss": 1.9028,
+      "step": 834
+    },
+    {
+      "epoch": 0.32270457164809835,
+      "grad_norm": 1.7286940813064575,
+      "learning_rate": 9.6655264442133e-05,
+      "loss": 1.8739,
+      "step": 840
+    },
+    {
+      "epoch": 0.32500960430272763,
+      "grad_norm": 1.7184300422668457,
+      "learning_rate": 9.658638812038379e-05,
+      "loss": 1.7968,
+      "step": 846
+    },
+    {
+      "epoch": 0.3273146369573569,
+      "grad_norm": 1.835468053817749,
+      "learning_rate": 9.651683489293797e-05,
+      "loss": 1.9953,
+      "step": 852
+    },
+    {
+      "epoch": 0.3296196696119862,
+      "grad_norm": 1.994387149810791,
+      "learning_rate": 9.644660577041214e-05,
+      "loss": 1.9847,
+      "step": 858
+    },
+    {
+      "epoch": 0.33192470226661547,
+      "grad_norm": 1.7545593976974487,
+      "learning_rate": 9.637570177324367e-05,
+      "loss": 1.8408,
+      "step": 864
+    },
+    {
+      "epoch": 0.3342297349212447,
+      "grad_norm": 2.230008363723755,
+      "learning_rate": 9.630412393167599e-05,
+      "loss": 1.9176,
+      "step": 870
+    },
+    {
+      "epoch": 0.33653476757587397,
+      "grad_norm": 1.968520998954773,
+      "learning_rate": 9.623187328574357e-05,
+      "loss": 1.8862,
+      "step": 876
+    },
+    {
+      "epoch": 0.33883980023050325,
+      "grad_norm": 1.8332586288452148,
+      "learning_rate": 9.615895088525677e-05,
+      "loss": 1.9738,
+      "step": 882
+    },
+    {
+      "epoch": 0.34114483288513253,
+      "grad_norm": 2.0475125312805176,
+      "learning_rate": 9.608535778978665e-05,
+      "loss": 1.9922,
+      "step": 888
+    },
+    {
+      "epoch": 0.3434498655397618,
+      "grad_norm": 1.7052634954452515,
+      "learning_rate": 9.601109506864953e-05,
+      "loss": 1.9191,
+      "step": 894
+    },
+    {
+      "epoch": 0.3457548981943911,
+      "grad_norm": 1.9678322076797485,
+      "learning_rate": 9.593616380089148e-05,
+      "loss": 1.9323,
+      "step": 900
+    },
+    {
+      "epoch": 0.34805993084902037,
+      "grad_norm": 1.7459259033203125,
+      "learning_rate": 9.586056507527266e-05,
+      "loss": 2.0201,
+      "step": 906
+    },
+    {
+      "epoch": 0.35036496350364965,
+      "grad_norm": 1.9217889308929443,
+      "learning_rate": 9.57842999902514e-05,
+      "loss": 2.0621,
+      "step": 912
+    },
+    {
+      "epoch": 0.35266999615827893,
+      "grad_norm": 2.16978120803833,
+      "learning_rate": 9.570736965396839e-05,
+      "loss": 1.9163,
+      "step": 918
+    },
+    {
+      "epoch": 0.35497502881290816,
+      "grad_norm": 1.6847466230392456,
+      "learning_rate": 9.562977518423042e-05,
+      "loss": 1.8978,
+      "step": 924
+    },
+    {
+      "epoch": 0.35728006146753744,
+      "grad_norm": 1.8538581132888794,
+      "learning_rate": 9.555151770849429e-05,
+      "loss": 1.854,
+      "step": 930
+    },
+    {
+      "epoch": 0.3595850941221667,
+      "grad_norm": 1.9544926881790161,
+      "learning_rate": 9.547259836385031e-05,
+      "loss": 1.9798,
+      "step": 936
+    },
+    {
+      "epoch": 0.361890126776796,
+      "grad_norm": 1.874722957611084,
+      "learning_rate": 9.539301829700584e-05,
+      "loss": 1.8237,
+      "step": 942
+    },
+    {
+      "epoch": 0.3641951594314253,
+      "grad_norm": 1.9412872791290283,
+      "learning_rate": 9.53127786642686e-05,
+      "loss": 1.9507,
+      "step": 948
+    },
+    {
+      "epoch": 0.36650019208605455,
+      "grad_norm": 1.826261043548584,
+      "learning_rate": 9.523188063152991e-05,
+      "loss": 1.9309,
+      "step": 954
+    },
+    {
+      "epoch": 0.36880522474068383,
+      "grad_norm": 1.716174602508545,
+      "learning_rate": 9.515032537424767e-05,
+      "loss": 1.8747,
+      "step": 960
+    },
+    {
+      "epoch": 0.3711102573953131,
+      "grad_norm": 1.6692321300506592,
+      "learning_rate": 9.506811407742937e-05,
+      "loss": 1.8537,
+      "step": 966
+    },
+    {
+      "epoch": 0.3734152900499424,
+      "grad_norm": 1.5984748601913452,
+      "learning_rate": 9.498524793561481e-05,
+      "loss": 1.855,
+      "step": 972
+    },
+    {
+      "epoch": 0.3757203227045717,
+      "grad_norm": 1.8006514310836792,
+      "learning_rate": 9.490172815285879e-05,
+      "loss": 1.936,
+      "step": 978
+    },
+    {
+      "epoch": 0.3780253553592009,
+      "grad_norm": 2.175570011138916,
+      "learning_rate": 9.481755594271356e-05,
+      "loss": 1.9957,
+      "step": 984
+    },
+    {
+      "epoch": 0.3803303880138302,
+      "grad_norm": 1.6217669248580933,
+      "learning_rate": 9.473273252821124e-05,
+      "loss": 1.9374,
+      "step": 990
+    },
+    {
+      "epoch": 0.38263542066845946,
+      "grad_norm": 1.905562400817871,
+      "learning_rate": 9.4647259141846e-05,
+      "loss": 1.8552,
+      "step": 996
+    },
+    {
+      "epoch": 0.38494045332308874,
+      "grad_norm": 2.333702325820923,
+      "learning_rate": 9.45611370255562e-05,
+      "loss": 1.8838,
+      "step": 1002
+    },
+    {
+      "epoch": 0.387245485977718,
+      "grad_norm": 1.9789848327636719,
+      "learning_rate": 9.447436743070632e-05,
+      "loss": 1.9141,
+      "step": 1008
+    },
+    {
+      "epoch": 0.3895505186323473,
+      "grad_norm": 1.9484792947769165,
+      "learning_rate": 9.438695161806877e-05,
+      "loss": 1.9527,
+      "step": 1014
+    },
+    {
+      "epoch": 0.3918555512869766,
+      "grad_norm": 2.220594644546509,
+      "learning_rate": 9.429889085780557e-05,
+      "loss": 1.8174,
+      "step": 1020
+    },
+    {
+      "epoch": 0.39416058394160586,
+      "grad_norm": 1.61322820186615,
+      "learning_rate": 9.421018642944997e-05,
+      "loss": 1.775,
+      "step": 1026
+    },
+    {
+      "epoch": 0.39646561659623514,
+      "grad_norm": 2.102656602859497,
+      "learning_rate": 9.412083962188768e-05,
+      "loss": 1.8479,
+      "step": 1032
+    },
+    {
+      "epoch": 0.39877064925086436,
+      "grad_norm": 2.0511844158172607,
+      "learning_rate": 9.403085173333836e-05,
+      "loss": 1.8995,
+      "step": 1038
+    },
+    {
+      "epoch": 0.40107568190549364,
+      "grad_norm": 1.809274673461914,
+      "learning_rate": 9.39402240713366e-05,
+      "loss": 1.8564,
+      "step": 1044
+    },
+    {
+      "epoch": 0.4033807145601229,
+      "grad_norm": 1.7851532697677612,
+      "learning_rate": 9.384895795271298e-05,
+      "loss": 1.8374,
+      "step": 1050
+    },
+    {
+      "epoch": 0.4056857472147522,
+      "grad_norm": 1.6801964044570923,
+      "learning_rate": 9.375705470357493e-05,
+      "loss": 1.9004,
+      "step": 1056
+    },
+    {
+      "epoch": 0.4079907798693815,
+      "grad_norm": 1.6246893405914307,
+      "learning_rate": 9.366451565928748e-05,
+      "loss": 1.8338,
+      "step": 1062
+    },
+    {
+      "epoch": 0.41029581252401076,
+      "grad_norm": 1.937282919883728,
+      "learning_rate": 9.35713421644538e-05,
+      "loss": 1.7568,
+      "step": 1068
+    },
+    {
+      "epoch": 0.41260084517864004,
+      "grad_norm": 1.6938468217849731,
+      "learning_rate": 9.347753557289575e-05,
+      "loss": 1.9216,
+      "step": 1074
+    },
+    {
+      "epoch": 0.4149058778332693,
+      "grad_norm": 1.902911901473999,
+      "learning_rate": 9.338309724763414e-05,
+      "loss": 1.7883,
+      "step": 1080
+    },
+    {
+      "epoch": 0.4172109104878986,
+      "grad_norm": 1.4669328927993774,
+      "learning_rate": 9.32880285608689e-05,
+      "loss": 1.8376,
+      "step": 1086
+    },
+    {
+      "epoch": 0.4195159431425278,
+      "grad_norm": 1.9248301982879639,
+      "learning_rate": 9.319233089395926e-05,
+      "loss": 1.7996,
+      "step": 1092
+    },
+    {
+      "epoch": 0.4218209757971571,
+      "grad_norm": 1.7929179668426514,
+      "learning_rate": 9.309600563740355e-05,
+      "loss": 1.9428,
+      "step": 1098
+    },
+    {
+      "epoch": 0.4241260084517864,
+      "grad_norm": 1.4758636951446533,
+      "learning_rate": 9.299905419081909e-05,
+      "loss": 1.7115,
+      "step": 1104
+    },
+    {
+      "epoch": 0.42643104110641566,
+      "grad_norm": 1.636285662651062,
+      "learning_rate": 9.290147796292179e-05,
+      "loss": 1.8874,
+      "step": 1110
+    },
+    {
+      "epoch": 0.42873607376104494,
+      "grad_norm": 1.589911699295044,
+      "learning_rate": 9.280327837150571e-05,
+      "loss": 1.8757,
+      "step": 1116
+    },
+    {
+      "epoch": 0.4310411064156742,
+      "grad_norm": 1.5797256231307983,
+      "learning_rate": 9.270445684342248e-05,
+      "loss": 1.9806,
+      "step": 1122
+    },
+    {
+      "epoch": 0.4333461390703035,
+      "grad_norm": 1.8015718460083008,
+      "learning_rate": 9.260501481456049e-05,
+      "loss": 1.8384,
+      "step": 1128
+    },
+    {
+      "epoch": 0.4356511717249328,
+      "grad_norm": 1.8427064418792725,
+      "learning_rate": 9.250495372982413e-05,
+      "loss": 1.7585,
+      "step": 1134
+    },
+    {
+      "epoch": 0.43795620437956206,
+      "grad_norm": 1.6950719356536865,
+      "learning_rate": 9.240427504311271e-05,
+      "loss": 1.8322,
+      "step": 1140
+    },
+    {
+      "epoch": 0.44026123703419134,
+      "grad_norm": 1.8371658325195312,
+      "learning_rate": 9.23029802172994e-05,
+      "loss": 1.9614,
+      "step": 1146
+    },
+    {
+      "epoch": 0.44256626968882057,
+      "grad_norm": 1.7700040340423584,
+      "learning_rate": 9.220107072420994e-05,
+      "loss": 1.8559,
+      "step": 1152
+    },
+    {
+      "epoch": 0.44487130234344985,
+      "grad_norm": 1.5368223190307617,
+      "learning_rate": 9.209854804460121e-05,
+      "loss": 1.902,
+      "step": 1158
+    },
+    {
+      "epoch": 0.44717633499807913,
+      "grad_norm": 1.7291918992996216,
+      "learning_rate": 9.199541366813982e-05,
+      "loss": 1.9065,
+      "step": 1164
+    },
+    {
+      "epoch": 0.4494813676527084,
+      "grad_norm": 1.5348674058914185,
+      "learning_rate": 9.189166909338041e-05,
+      "loss": 1.9273,
+      "step": 1170
+    },
+    {
+      "epoch": 0.4517864003073377,
+      "grad_norm": 1.935111403465271,
+      "learning_rate": 9.178731582774381e-05,
+      "loss": 1.7897,
+      "step": 1176
+    },
+    {
+      "epoch": 0.45409143296196697,
+      "grad_norm": 1.6748031377792358,
+      "learning_rate": 9.168235538749524e-05,
+      "loss": 1.7911,
+      "step": 1182
+    },
+    {
+      "epoch": 0.45639646561659625,
+      "grad_norm": 2.3855085372924805,
+      "learning_rate": 9.157678929772224e-05,
+      "loss": 1.952,
+      "step": 1188
+    },
+    {
+      "epoch": 0.4587014982712255,
+      "grad_norm": 1.680123209953308,
+      "learning_rate": 9.147061909231253e-05,
+      "loss": 1.8363,
+      "step": 1194
+    },
+    {
+      "epoch": 0.4610065309258548,
+      "grad_norm": 1.7553462982177734,
+      "learning_rate": 9.136384631393167e-05,
+      "loss": 1.8989,
+      "step": 1200
+    },
+    {
+      "epoch": 0.46331156358048403,
+      "grad_norm": 1.5558507442474365,
+      "learning_rate": 9.125647251400067e-05,
+      "loss": 1.8772,
+      "step": 1206
+    },
+    {
+      "epoch": 0.4656165962351133,
+      "grad_norm": 1.7176810503005981,
+      "learning_rate": 9.114849925267345e-05,
+      "loss": 1.7789,
+      "step": 1212
+    },
+    {
+      "epoch": 0.4679216288897426,
+      "grad_norm": 1.7112728357315063,
+      "learning_rate": 9.103992809881425e-05,
+      "loss": 1.8281,
+      "step": 1218
+    },
+    {
+      "epoch": 0.47022666154437187,
+      "grad_norm": 1.5937122106552124,
+      "learning_rate": 9.093076062997465e-05,
+      "loss": 1.9652,
+      "step": 1224
+    },
+    {
+      "epoch": 0.47253169419900115,
+      "grad_norm": 1.8307805061340332,
+      "learning_rate": 9.082099843237081e-05,
+      "loss": 1.9037,
+      "step": 1230
+    },
+    {
+      "epoch": 0.47483672685363043,
+      "grad_norm": 1.7895787954330444,
+      "learning_rate": 9.07106431008604e-05,
+      "loss": 1.7806,
+      "step": 1236
+    },
+    {
+      "epoch": 0.4771417595082597,
+      "grad_norm": 1.6272079944610596,
+      "learning_rate": 9.059969623891932e-05,
+      "loss": 1.8164,
+      "step": 1242
+    },
+    {
+      "epoch": 0.479446792162889,
+      "grad_norm": 1.609658122062683,
+      "learning_rate": 9.048815945861854e-05,
+      "loss": 1.8687,
+      "step": 1248
+    },
+    {
+      "epoch": 0.48175182481751827,
+      "grad_norm": 1.6670969724655151,
+      "learning_rate": 9.037603438060058e-05,
+      "loss": 1.8342,
+      "step": 1254
+    },
+    {
+      "epoch": 0.4840568574721475,
+      "grad_norm": 1.9929320812225342,
+      "learning_rate": 9.026332263405604e-05,
+      "loss": 1.8555,
+      "step": 1260
+    },
+    {
+      "epoch": 0.4863618901267768,
+      "grad_norm": 1.849782109260559,
+      "learning_rate": 9.01500258566998e-05,
+      "loss": 1.8635,
+      "step": 1266
+    },
+    {
+      "epoch": 0.48866692278140605,
+      "grad_norm": 1.5585784912109375,
+      "learning_rate": 9.003614569474738e-05,
+      "loss": 1.8222,
+      "step": 1272
+    },
+    {
+      "epoch": 0.49097195543603533,
+      "grad_norm": 1.8432416915893555,
+      "learning_rate": 8.992168380289095e-05,
+      "loss": 1.8082,
+      "step": 1278
+    },
+    {
+      "epoch": 0.4932769880906646,
+      "grad_norm": 1.6949557065963745,
+      "learning_rate": 8.980664184427518e-05,
+      "loss": 1.8169,
+      "step": 1284
+    },
+    {
+      "epoch": 0.4955820207452939,
+      "grad_norm": 1.7418153285980225,
+      "learning_rate": 8.969102149047333e-05,
+      "loss": 1.8293,
+      "step": 1290
+    },
+    {
+      "epoch": 0.4978870533999232,
+      "grad_norm": 1.9884105920791626,
+      "learning_rate": 8.957482442146272e-05,
+      "loss": 1.7974,
+      "step": 1296
+    },
+    {
+      "epoch": 0.5001920860545525,
+      "grad_norm": 1.9336163997650146,
+      "learning_rate": 8.945805232560038e-05,
+      "loss": 1.8266,
+      "step": 1302
+    },
+    {
+      "epoch": 0.5024971187091817,
+      "grad_norm": 1.9202194213867188,
+      "learning_rate": 8.934070689959864e-05,
+      "loss": 1.8011,
+      "step": 1308
+    },
+    {
+      "epoch": 0.504802151363811,
+      "grad_norm": 1.5891252756118774,
+      "learning_rate": 8.92227898485003e-05,
+      "loss": 1.7762,
+      "step": 1314
+    },
+    {
+      "epoch": 0.5071071840184402,
+      "grad_norm": 1.5969560146331787,
+      "learning_rate": 8.910430288565401e-05,
+      "loss": 1.9395,
+      "step": 1320
+    },
+    {
+      "epoch": 0.5094122166730696,
+      "grad_norm": 1.4814295768737793,
+      "learning_rate": 8.898524773268926e-05,
+      "loss": 1.8461,
+      "step": 1326
+    },
+    {
+      "epoch": 0.5117172493276988,
+      "grad_norm": 1.936658263206482,
+      "learning_rate": 8.88656261194914e-05,
+      "loss": 1.8454,
+      "step": 1332
+    },
+    {
+      "epoch": 0.5140222819823281,
+      "grad_norm": 2.066612958908081,
+      "learning_rate": 8.874543978417657e-05,
+      "loss": 1.7764,
+      "step": 1338
+    },
+    {
+      "epoch": 0.5163273146369574,
+      "grad_norm": 1.916707992553711,
+      "learning_rate": 8.862469047306632e-05,
+      "loss": 1.8664,
+      "step": 1344
+    },
+    {
+      "epoch": 0.5186323472915866,
+      "grad_norm": 1.631078839302063,
+      "learning_rate": 8.850337994066234e-05,
+      "loss": 1.806,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5209373799462159,
+      "grad_norm": 1.783075213432312,
+      "learning_rate": 8.838150994962093e-05,
+      "loss": 1.8374,
+      "step": 1356
+    },
+    {
+      "epoch": 0.5232424126008451,
+      "grad_norm": 1.5385944843292236,
+      "learning_rate": 8.825908227072742e-05,
+      "loss": 1.7789,
+      "step": 1362
+    },
+    {
+      "epoch": 0.5255474452554745,
+      "grad_norm": 1.9199451208114624,
+      "learning_rate": 8.813609868287034e-05,
+      "loss": 1.8132,
+      "step": 1368
+    },
+    {
+      "epoch": 0.5278524779101037,
+      "grad_norm": 1.961859107017517,
+      "learning_rate": 8.801256097301569e-05,
+      "loss": 1.8908,
+      "step": 1374
+    },
+    {
+      "epoch": 0.530157510564733,
+      "grad_norm": 1.4744768142700195,
+      "learning_rate": 8.788847093618093e-05,
+      "loss": 1.6905,
+      "step": 1380
+    },
+    {
+      "epoch": 0.5324625432193623,
+      "grad_norm": 1.7731544971466064,
+      "learning_rate": 8.776383037540888e-05,
+      "loss": 1.6969,
+      "step": 1386
+    },
+    {
+      "epoch": 0.5347675758739916,
+      "grad_norm": 1.7354005575180054,
+      "learning_rate": 8.763864110174154e-05,
+      "loss": 1.7821,
+      "step": 1392
+    },
+    {
+      "epoch": 0.5370726085286208,
+      "grad_norm": 1.4063547849655151,
+      "learning_rate": 8.751290493419377e-05,
+      "loss": 1.7083,
+      "step": 1398
+    },
+    {
+      "epoch": 0.53937764118325,
+      "grad_norm": 1.5864259004592896,
+      "learning_rate": 8.738662369972687e-05,
+      "loss": 1.7859,
+      "step": 1404
+    },
+    {
+      "epoch": 0.5416826738378794,
+      "grad_norm": 1.6502201557159424,
+      "learning_rate": 8.725979923322203e-05,
+      "loss": 1.7355,
+      "step": 1410
+    },
+    {
+      "epoch": 0.5439877064925086,
+      "grad_norm": 1.6681023836135864,
+      "learning_rate": 8.713243337745365e-05,
+      "loss": 1.8342,
+      "step": 1416
+    },
+    {
+      "epoch": 0.5462927391471379,
+      "grad_norm": 1.5387425422668457,
+      "learning_rate": 8.70045279830626e-05,
+      "loss": 1.7961,
+      "step": 1422
+    },
+    {
+      "epoch": 0.5485977718017672,
+      "grad_norm": 1.5881112813949585,
+      "learning_rate": 8.687608490852929e-05,
+      "loss": 1.832,
+      "step": 1428
+    },
+    {
+      "epoch": 0.5509028044563965,
+      "grad_norm": 1.9706406593322754,
+      "learning_rate": 8.674710602014671e-05,
+      "loss": 1.7834,
+      "step": 1434
+    },
+    {
+      "epoch": 0.5532078371110257,
+      "grad_norm": 1.7211198806762695,
+      "learning_rate": 8.661759319199323e-05,
+      "loss": 1.7126,
+      "step": 1440
+    },
+    {
+      "epoch": 0.5555128697656551,
+      "grad_norm": 1.650084137916565,
+      "learning_rate": 8.648754830590552e-05,
+      "loss": 1.8515,
+      "step": 1446
+    },
+    {
+      "epoch": 0.5578179024202843,
+      "grad_norm": 1.5930005311965942,
+      "learning_rate": 8.635697325145104e-05,
+      "loss": 1.8805,
+      "step": 1452
+    },
+    {
+      "epoch": 0.5601229350749135,
+      "grad_norm": 1.786905288696289,
+      "learning_rate": 8.622586992590069e-05,
+      "loss": 1.7593,
+      "step": 1458
+    },
+    {
+      "epoch": 0.5624279677295428,
+      "grad_norm": 1.7052221298217773,
+      "learning_rate": 8.609424023420119e-05,
+      "loss": 1.7634,
+      "step": 1464
+    },
+    {
+      "epoch": 0.5647330003841721,
+      "grad_norm": 1.5389682054519653,
+      "learning_rate": 8.596208608894745e-05,
+      "loss": 1.7536,
+      "step": 1470
+    },
+    {
+      "epoch": 0.5670380330388014,
+      "grad_norm": 1.6341835260391235,
+      "learning_rate": 8.582940941035476e-05,
+      "loss": 1.8715,
+      "step": 1476
+    },
+    {
+      "epoch": 0.5693430656934306,
+      "grad_norm": 1.7053039073944092,
+      "learning_rate": 8.569621212623081e-05,
+      "loss": 1.855,
+      "step": 1482
+    },
+    {
+      "epoch": 0.57164809834806,
+      "grad_norm": 1.883748173713684,
+      "learning_rate": 8.556249617194781e-05,
+      "loss": 1.8449,
+      "step": 1488
+    },
+    {
+      "epoch": 0.5739531310026892,
+      "grad_norm": 1.689418911933899,
+      "learning_rate": 8.542826349041432e-05,
+      "loss": 1.8314,
+      "step": 1494
+    },
+    {
+      "epoch": 0.5762581636573185,
+      "grad_norm": 1.6523135900497437,
+      "learning_rate": 8.529351603204698e-05,
+      "loss": 1.8284,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5785631963119477,
+      "grad_norm": 1.6415992975234985,
+      "learning_rate": 8.515825575474221e-05,
+      "loss": 1.9305,
+      "step": 1506
+    },
+    {
+      "epoch": 0.580868228966577,
+      "grad_norm": 1.7368993759155273,
+      "learning_rate": 8.50224846238477e-05,
+      "loss": 1.8213,
+      "step": 1512
+    },
+    {
+      "epoch": 0.5831732616212063,
+      "grad_norm": 1.4232181310653687,
+      "learning_rate": 8.488620461213399e-05,
+      "loss": 1.8164,
+      "step": 1518
+    },
+    {
+      "epoch": 0.5854782942758355,
+      "grad_norm": 1.737285852432251,
+      "learning_rate": 8.474941769976566e-05,
+      "loss": 1.7534,
+      "step": 1524
+    },
+    {
+      "epoch": 0.5877833269304649,
+      "grad_norm": 1.6582894325256348,
+      "learning_rate": 8.46121258742726e-05,
+      "loss": 1.8482,
+      "step": 1530
+    },
+    {
+      "epoch": 0.5900883595850941,
+      "grad_norm": 1.9711023569107056,
+      "learning_rate": 8.447433113052123e-05,
+      "loss": 1.7701,
+      "step": 1536
+    },
+    {
+      "epoch": 0.5923933922397234,
+      "grad_norm": 1.75121009349823,
+      "learning_rate": 8.433603547068534e-05,
+      "loss": 1.8797,
+      "step": 1542
+    },
+    {
+      "epoch": 0.5946984248943527,
+      "grad_norm": 1.6199707984924316,
+      "learning_rate": 8.419724090421713e-05,
+      "loss": 1.7395,
+      "step": 1548
+    },
+    {
+      "epoch": 0.597003457548982,
+      "grad_norm": 2.1800003051757812,
+      "learning_rate": 8.405794944781797e-05,
+      "loss": 1.7486,
+      "step": 1554
+    },
+    {
+      "epoch": 0.5993084902036112,
+      "grad_norm": 1.6844942569732666,
+      "learning_rate": 8.39181631254091e-05,
+      "loss": 1.8276,
+      "step": 1560
+    },
+    {
+      "epoch": 0.6016135228582405,
+      "grad_norm": 1.5212658643722534,
+      "learning_rate": 8.377788396810223e-05,
+      "loss": 1.7147,
+      "step": 1566
+    },
+    {
+      "epoch": 0.6039185555128698,
+      "grad_norm": 1.6235482692718506,
+      "learning_rate": 8.363711401417e-05,
+      "loss": 1.6997,
+      "step": 1572
+    },
+    {
+      "epoch": 0.606223588167499,
+      "grad_norm": 1.5753910541534424,
+      "learning_rate": 8.349585530901639e-05,
+      "loss": 1.7416,
+      "step": 1578
+    },
+    {
+      "epoch": 0.6085286208221283,
+      "grad_norm": 2.090818166732788,
+      "learning_rate": 8.3354109905147e-05,
+      "loss": 1.7774,
+      "step": 1584
+    },
+    {
+      "epoch": 0.6108336534767576,
+      "grad_norm": 1.416100263595581,
+      "learning_rate": 8.32118798621392e-05,
+      "loss": 1.8276,
+      "step": 1590
+    },
+    {
+      "epoch": 0.6131386861313869,
+      "grad_norm": 1.5067269802093506,
+      "learning_rate": 8.306916724661224e-05,
+      "loss": 1.8266,
+      "step": 1596
+    },
+    {
+      "epoch": 0.6154437187860161,
+      "grad_norm": 2.855164051055908,
+      "learning_rate": 8.292597413219723e-05,
+      "loss": 1.7834,
+      "step": 1602
+    },
+    {
+      "epoch": 0.6177487514406454,
+      "grad_norm": 1.5710291862487793,
+      "learning_rate": 8.278230259950693e-05,
+      "loss": 1.7725,
+      "step": 1608
+    },
+    {
+      "epoch": 0.6200537840952747,
+      "grad_norm": 1.541718602180481,
+      "learning_rate": 8.263815473610559e-05,
+      "loss": 1.7741,
+      "step": 1614
+    },
+    {
+      "epoch": 0.622358816749904,
+      "grad_norm": 1.5476350784301758,
+      "learning_rate": 8.249353263647864e-05,
+      "loss": 1.833,
+      "step": 1620
+    },
+    {
+      "epoch": 0.6246638494045332,
+      "grad_norm": 1.6447367668151855,
+      "learning_rate": 8.234843840200217e-05,
+      "loss": 1.794,
+      "step": 1626
+    },
+    {
+      "epoch": 0.6269688820591625,
+      "grad_norm": 1.865459680557251,
+      "learning_rate": 8.22028741409125e-05,
+      "loss": 1.6391,
+      "step": 1632
+    },
+    {
+      "epoch": 0.6292739147137918,
+      "grad_norm": 1.4397554397583008,
+      "learning_rate": 8.205684196827542e-05,
+      "loss": 1.8012,
+      "step": 1638
+    },
+    {
+      "epoch": 0.631578947368421,
+      "grad_norm": 1.6148408651351929,
+      "learning_rate": 8.191034400595559e-05,
+      "loss": 1.8852,
+      "step": 1644
+    },
+    {
+      "epoch": 0.6338839800230504,
+      "grad_norm": 1.6250483989715576,
+      "learning_rate": 8.17633823825856e-05,
+      "loss": 1.8188,
+      "step": 1650
+    },
+    {
+      "epoch": 0.6361890126776796,
+      "grad_norm": 1.435920000076294,
+      "learning_rate": 8.161595923353516e-05,
+      "loss": 1.9247,
+      "step": 1656
+    },
+    {
+      "epoch": 0.6384940453323089,
+      "grad_norm": 1.8216177225112915,
+      "learning_rate": 8.146807670087993e-05,
+      "loss": 1.7493,
+      "step": 1662
+    },
+    {
+      "epoch": 0.6407990779869381,
+      "grad_norm": 1.687909483909607,
+      "learning_rate": 8.131973693337049e-05,
+      "loss": 1.8061,
+      "step": 1668
+    },
+    {
+      "epoch": 0.6431041106415675,
+      "grad_norm": 1.6617321968078613,
+      "learning_rate": 8.11709420864011e-05,
+      "loss": 1.925,
+      "step": 1674
+    },
+    {
+      "epoch": 0.6454091432961967,
+      "grad_norm": 1.7653294801712036,
+      "learning_rate": 8.102169432197842e-05,
+      "loss": 1.7659,
+      "step": 1680
+    },
+    {
+      "epoch": 0.6477141759508259,
+      "grad_norm": 1.7562230825424194,
+      "learning_rate": 8.087199580868996e-05,
+      "loss": 1.7817,
+      "step": 1686
+    },
+    {
+      "epoch": 0.6500192086054553,
+      "grad_norm": 1.5609056949615479,
+      "learning_rate": 8.072184872167275e-05,
+      "loss": 1.861,
+      "step": 1692
+    },
+    {
+      "epoch": 0.6523242412600845,
+      "grad_norm": 1.6291929483413696,
+      "learning_rate": 8.057125524258163e-05,
+      "loss": 1.7036,
+      "step": 1698
+    },
+    {
+      "epoch": 0.6546292739147138,
+      "grad_norm": 1.7832359075546265,
+      "learning_rate": 8.042021755955752e-05,
+      "loss": 1.7953,
+      "step": 1704
+    },
+    {
+      "epoch": 0.656934306569343,
+      "grad_norm": 1.5798019170761108,
+      "learning_rate": 8.026873786719573e-05,
+      "loss": 1.7061,
+      "step": 1710
+    },
+    {
+      "epoch": 0.6592393392239724,
+      "grad_norm": 1.4281039237976074,
+      "learning_rate": 8.011681836651402e-05,
+      "loss": 1.7173,
+      "step": 1716
+    },
+    {
+      "epoch": 0.6615443718786016,
+      "grad_norm": 1.418607234954834,
+      "learning_rate": 7.996446126492057e-05,
+      "loss": 1.7646,
+      "step": 1722
+    },
+    {
+      "epoch": 0.6638494045332309,
+      "grad_norm": 1.9649653434753418,
+      "learning_rate": 7.981166877618197e-05,
+      "loss": 1.7006,
+      "step": 1728
+    },
+    {
+      "epoch": 0.6661544371878602,
+      "grad_norm": 1.9355838298797607,
+      "learning_rate": 7.965844312039108e-05,
+      "loss": 1.8712,
+      "step": 1734
+    },
+    {
+      "epoch": 0.6684594698424894,
+      "grad_norm": 1.7846251726150513,
+      "learning_rate": 7.950478652393466e-05,
+      "loss": 1.7425,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6707645024971187,
+      "grad_norm": 1.5747418403625488,
+      "learning_rate": 7.935070121946115e-05,
+      "loss": 1.8506,
+      "step": 1746
+    },
+    {
+      "epoch": 0.6730695351517479,
+      "grad_norm": 1.5762399435043335,
+      "learning_rate": 7.919618944584812e-05,
+      "loss": 1.8565,
+      "step": 1752
+    },
+    {
+      "epoch": 0.6753745678063773,
+      "grad_norm": 1.485330581665039,
+      "learning_rate": 7.904125344816982e-05,
+      "loss": 1.8489,
+      "step": 1758
+    },
+    {
+      "epoch": 0.6776796004610065,
+      "grad_norm": 1.515936255455017,
+      "learning_rate": 7.888589547766449e-05,
+      "loss": 1.7496,
+      "step": 1764
+    },
+    {
+      "epoch": 0.6799846331156358,
+      "grad_norm": 1.3050755262374878,
+      "learning_rate": 7.873011779170173e-05,
+      "loss": 1.7325,
+      "step": 1770
+    },
+    {
+      "epoch": 0.6822896657702651,
+      "grad_norm": 1.353097677230835,
+      "learning_rate": 7.857392265374963e-05,
+      "loss": 1.7661,
+      "step": 1776
+    },
+    {
+      "epoch": 0.6845946984248944,
+      "grad_norm": 1.5396977663040161,
+      "learning_rate": 7.84173123333419e-05,
+      "loss": 1.6669,
+      "step": 1782
+    },
+    {
+      "epoch": 0.6868997310795236,
+      "grad_norm": 1.4512056112289429,
+      "learning_rate": 7.82602891060449e-05,
+      "loss": 1.7954,
+      "step": 1788
+    },
+    {
+      "epoch": 0.6892047637341528,
+      "grad_norm": 1.48707115650177,
+      "learning_rate": 7.81028552534246e-05,
+      "loss": 1.8151,
+      "step": 1794
+    },
+    {
+      "epoch": 0.6915097963887822,
+      "grad_norm": 1.6403846740722656,
+      "learning_rate": 7.794501306301341e-05,
+      "loss": 1.7789,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6938148290434114,
+      "grad_norm": 1.742187738418579,
+      "learning_rate": 7.778676482827686e-05,
+      "loss": 1.6794,
+      "step": 1806
+    },
+    {
+      "epoch": 0.6961198616980407,
+      "grad_norm": 1.5124804973602295,
+      "learning_rate": 7.762811284858045e-05,
+      "loss": 1.7103,
+      "step": 1812
+    },
+    {
+      "epoch": 0.69842489435267,
+      "grad_norm": 1.6683412790298462,
+      "learning_rate": 7.746905942915608e-05,
+      "loss": 1.8293,
+      "step": 1818
+    },
+    {
+      "epoch": 0.7007299270072993,
+      "grad_norm": 1.549338698387146,
+      "learning_rate": 7.730960688106861e-05,
+      "loss": 1.804,
+      "step": 1824
+    },
+    {
+      "epoch": 0.7030349596619285,
+      "grad_norm": 1.552217721939087,
+      "learning_rate": 7.714975752118237e-05,
+      "loss": 1.7282,
+      "step": 1830
+    },
+    {
+      "epoch": 0.7053399923165579,
+      "grad_norm": 1.6498384475708008,
+      "learning_rate": 7.69895136721273e-05,
+      "loss": 1.7662,
+      "step": 1836
+    },
+    {
+      "epoch": 0.7076450249711871,
+      "grad_norm": 1.450636863708496,
+      "learning_rate": 7.682887766226539e-05,
+      "loss": 1.6994,
+      "step": 1842
+    },
+    {
+      "epoch": 0.7099500576258163,
+      "grad_norm": 1.6850075721740723,
+      "learning_rate": 7.666785182565677e-05,
+      "loss": 1.7617,
+      "step": 1848
+    },
+    {
+      "epoch": 0.7122550902804456,
+      "grad_norm": 1.4256203174591064,
+      "learning_rate": 7.650643850202572e-05,
+      "loss": 1.7773,
+      "step": 1854
+    },
+    {
+      "epoch": 0.7145601229350749,
+      "grad_norm": 1.636516809463501,
+      "learning_rate": 7.634464003672685e-05,
+      "loss": 1.8225,
+      "step": 1860
+    },
+    {
+      "epoch": 0.7168651555897042,
+      "grad_norm": 1.6618921756744385,
+      "learning_rate": 7.618245878071091e-05,
+      "loss": 1.6029,
+      "step": 1866
+    },
+    {
+      "epoch": 0.7191701882443334,
+      "grad_norm": 1.4596660137176514,
+      "learning_rate": 7.601989709049056e-05,
+      "loss": 1.7718,
+      "step": 1872
+    },
+    {
+      "epoch": 0.7214752208989628,
+      "grad_norm": 1.5597732067108154,
+      "learning_rate": 7.585695732810629e-05,
+      "loss": 1.9228,
+      "step": 1878
+    },
+    {
+      "epoch": 0.723780253553592,
+      "grad_norm": 1.7617650032043457,
+      "learning_rate": 7.569364186109202e-05,
+      "loss": 1.7215,
+      "step": 1884
+    },
+    {
+      "epoch": 0.7260852862082213,
+      "grad_norm": 1.540544033050537,
+      "learning_rate": 7.552995306244068e-05,
+      "loss": 1.7924,
+      "step": 1890
+    },
+    {
+      "epoch": 0.7283903188628505,
+      "grad_norm": 1.8182754516601562,
+      "learning_rate": 7.536589331056976e-05,
+      "loss": 1.792,
+      "step": 1896
+    },
+    {
+      "epoch": 0.7306953515174799,
+      "grad_norm": 1.6059759855270386,
+      "learning_rate": 7.520146498928675e-05,
+      "loss": 1.713,
+      "step": 1902
+    },
+    {
+      "epoch": 0.7330003841721091,
+      "grad_norm": 1.6662769317626953,
+      "learning_rate": 7.503667048775447e-05,
+      "loss": 1.7883,
+      "step": 1908
+    },
+    {
+      "epoch": 0.7353054168267383,
+      "grad_norm": 1.472133994102478,
+      "learning_rate": 7.48715122004564e-05,
+      "loss": 1.6923,
+      "step": 1914
+    },
+    {
+      "epoch": 0.7376104494813677,
+      "grad_norm": 1.4528474807739258,
+      "learning_rate": 7.470599252716191e-05,
+      "loss": 1.618,
+      "step": 1920
+    },
+    {
+      "epoch": 0.7399154821359969,
+      "grad_norm": 1.5965889692306519,
+      "learning_rate": 7.454011387289127e-05,
+      "loss": 1.8486,
+      "step": 1926
+    },
+    {
+      "epoch": 0.7422205147906262,
+      "grad_norm": 1.4224053621292114,
+      "learning_rate": 7.437387864788085e-05,
+      "loss": 1.6868,
+      "step": 1932
+    },
+    {
+      "epoch": 0.7445255474452555,
+      "grad_norm": 1.668214201927185,
+      "learning_rate": 7.420728926754803e-05,
+      "loss": 1.7565,
+      "step": 1938
+    },
+    {
+      "epoch": 0.7468305800998848,
+      "grad_norm": 1.6973024606704712,
+      "learning_rate": 7.404034815245607e-05,
+      "loss": 1.7967,
+      "step": 1944
+    },
+    {
+      "epoch": 0.749135612754514,
+      "grad_norm": 1.525120735168457,
+      "learning_rate": 7.387305772827899e-05,
+      "loss": 1.7815,
+      "step": 1950
+    },
+    {
+      "epoch": 0.7514406454091433,
+      "grad_norm": 1.5095804929733276,
+      "learning_rate": 7.370542042576634e-05,
+      "loss": 1.7344,
+      "step": 1956
+    },
+    {
+      "epoch": 0.7537456780637726,
+      "grad_norm": 1.6638365983963013,
+      "learning_rate": 7.353743868070785e-05,
+      "loss": 1.6713,
+      "step": 1962
+    },
+    {
+      "epoch": 0.7560507107184018,
+      "grad_norm": 1.6456612348556519,
+      "learning_rate": 7.336911493389799e-05,
+      "loss": 1.8417,
+      "step": 1968
+    },
+    {
+      "epoch": 0.7583557433730311,
+      "grad_norm": 1.7696939706802368,
+      "learning_rate": 7.32004516311006e-05,
+      "loss": 1.7315,
+      "step": 1974
+    },
+    {
+      "epoch": 0.7606607760276604,
+      "grad_norm": 1.5991597175598145,
+      "learning_rate": 7.303145122301331e-05,
+      "loss": 1.7376,
+      "step": 1980
+    },
+    {
+      "epoch": 0.7629658086822897,
+      "grad_norm": 1.5692775249481201,
+      "learning_rate": 7.286211616523193e-05,
+      "loss": 1.7466,
+      "step": 1986
+    },
+    {
+      "epoch": 0.7652708413369189,
+      "grad_norm": 1.8568308353424072,
+      "learning_rate": 7.269244891821474e-05,
+      "loss": 1.7878,
+      "step": 1992
+    },
+    {
+      "epoch": 0.7675758739915483,
+      "grad_norm": 1.365517258644104,
+      "learning_rate": 7.252245194724678e-05,
+      "loss": 1.7881,
+      "step": 1998
+    },
+    {
+      "epoch": 0.7698809066461775,
+      "grad_norm": 1.7684406042099,
+      "learning_rate": 7.235212772240405e-05,
+      "loss": 1.8298,
+      "step": 2004
+    },
+    {
+      "epoch": 0.7721859393008068,
+      "grad_norm": 1.4185709953308105,
+      "learning_rate": 7.218147871851755e-05,
+      "loss": 1.7461,
+      "step": 2010
+    },
+    {
+      "epoch": 0.774490971955436,
+      "grad_norm": 1.6174180507659912,
+      "learning_rate": 7.201050741513736e-05,
+      "loss": 1.7367,
+      "step": 2016
+    },
+    {
+      "epoch": 0.7767960046100653,
+      "grad_norm": 1.378377914428711,
+      "learning_rate": 7.183921629649664e-05,
+      "loss": 1.7649,
+      "step": 2022
+    },
+    {
+      "epoch": 0.7791010372646946,
+      "grad_norm": 1.6113669872283936,
+      "learning_rate": 7.166760785147546e-05,
+      "loss": 1.92,
+      "step": 2028
+    },
+    {
+      "epoch": 0.7814060699193238,
+      "grad_norm": 1.492136001586914,
+      "learning_rate": 7.149568457356471e-05,
+      "loss": 1.7296,
+      "step": 2034
+    },
+    {
+      "epoch": 0.7837111025739532,
+      "grad_norm": 2.28481125831604,
+      "learning_rate": 7.132344896082984e-05,
+      "loss": 1.7528,
+      "step": 2040
+    },
+    {
+      "epoch": 0.7860161352285824,
+      "grad_norm": 1.3087348937988281,
+      "learning_rate": 7.115090351587454e-05,
+      "loss": 1.7281,
+      "step": 2046
+    },
+    {
+      "epoch": 0.7883211678832117,
+      "grad_norm": 1.8530024290084839,
+      "learning_rate": 7.097805074580443e-05,
+      "loss": 1.7676,
+      "step": 2052
+    },
+    {
+      "epoch": 0.7906262005378409,
+      "grad_norm": 1.681631326675415,
+      "learning_rate": 7.080489316219058e-05,
+      "loss": 1.7009,
+      "step": 2058
+    },
+    {
+      "epoch": 0.7929312331924703,
+      "grad_norm": 1.6794466972351074,
+      "learning_rate": 7.063143328103302e-05,
+      "loss": 1.7374,
+      "step": 2064
+    },
+    {
+      "epoch": 0.7952362658470995,
+      "grad_norm": 1.599555492401123,
+      "learning_rate": 7.045767362272422e-05,
+      "loss": 1.6902,
+      "step": 2070
+    },
+    {
+      "epoch": 0.7975412985017287,
+      "grad_norm": 1.586045742034912,
+      "learning_rate": 7.028361671201245e-05,
+      "loss": 1.7369,
+      "step": 2076
+    },
+    {
+      "epoch": 0.7998463311563581,
+      "grad_norm": 1.7884392738342285,
+      "learning_rate": 7.010926507796509e-05,
+      "loss": 1.7825,
+      "step": 2082
+    },
+    {
+      "epoch": 0.8021513638109873,
+      "grad_norm": 1.8107120990753174,
+      "learning_rate": 6.993462125393188e-05,
+      "loss": 1.7774,
+      "step": 2088
+    },
+    {
+      "epoch": 0.8044563964656166,
+      "grad_norm": 1.480247974395752,
+      "learning_rate": 6.975968777750812e-05,
+      "loss": 1.756,
+      "step": 2094
+    },
+    {
+      "epoch": 0.8067614291202458,
+      "grad_norm": 1.4874813556671143,
+      "learning_rate": 6.958446719049777e-05,
+      "loss": 1.7013,
+      "step": 2100
+    },
+    {
+      "epoch": 0.8090664617748752,
+      "grad_norm": 1.6244595050811768,
+      "learning_rate": 6.940896203887659e-05,
+      "loss": 1.7122,
+      "step": 2106
+    },
+    {
+      "epoch": 0.8113714944295044,
+      "grad_norm": 1.3391255140304565,
+      "learning_rate": 6.923317487275505e-05,
+      "loss": 1.6446,
+      "step": 2112
+    },
+    {
+      "epoch": 0.8136765270841337,
+      "grad_norm": 1.7652738094329834,
+      "learning_rate": 6.905710824634133e-05,
+      "loss": 1.7361,
+      "step": 2118
+    },
+    {
+      "epoch": 0.815981559738763,
+      "grad_norm": 1.5222631692886353,
+      "learning_rate": 6.888076471790424e-05,
+      "loss": 1.6091,
+      "step": 2124
+    },
+    {
+      "epoch": 0.8182865923933922,
+      "grad_norm": 1.573530912399292,
+      "learning_rate": 6.870414684973594e-05,
+      "loss": 1.7937,
+      "step": 2130
+    },
+    {
+      "epoch": 0.8205916250480215,
+      "grad_norm": 1.5537464618682861,
+      "learning_rate": 6.852725720811488e-05,
+      "loss": 1.7491,
+      "step": 2136
+    },
+    {
+      "epoch": 0.8228966577026507,
+      "grad_norm": 1.7604540586471558,
+      "learning_rate": 6.83500983632683e-05,
+      "loss": 1.6455,
+      "step": 2142
+    },
+    {
+      "epoch": 0.8252016903572801,
+      "grad_norm": 1.4649218320846558,
+      "learning_rate": 6.817267288933507e-05,
+      "loss": 1.75,
+      "step": 2148
+    },
+    {
+      "epoch": 0.8275067230119093,
+      "grad_norm": 1.4161078929901123,
+      "learning_rate": 6.799498336432816e-05,
+      "loss": 1.869,
+      "step": 2154
+    },
+    {
+      "epoch": 0.8298117556665386,
+      "grad_norm": 1.6333036422729492,
+      "learning_rate": 6.781703237009728e-05,
+      "loss": 1.7483,
+      "step": 2160
+    },
+    {
+      "epoch": 0.8321167883211679,
+      "grad_norm": 1.2524577379226685,
+      "learning_rate": 6.76388224922913e-05,
+      "loss": 1.6337,
+      "step": 2166
+    },
+    {
+      "epoch": 0.8344218209757972,
+      "grad_norm": 1.5197899341583252,
+      "learning_rate": 6.746035632032068e-05,
+      "loss": 1.6982,
+      "step": 2172
+    },
+    {
+      "epoch": 0.8367268536304264,
+      "grad_norm": 1.6081801652908325,
+      "learning_rate": 6.72816364473199e-05,
+      "loss": 1.7803,
+      "step": 2178
+    },
+    {
+      "epoch": 0.8390318862850557,
+      "grad_norm": 1.4385576248168945,
+      "learning_rate": 6.710266547010973e-05,
+      "loss": 1.7166,
+      "step": 2184
+    },
+    {
+      "epoch": 0.841336918939685,
+      "grad_norm": 1.5211519002914429,
+      "learning_rate": 6.692344598915949e-05,
+      "loss": 1.6935,
+      "step": 2190
+    },
+    {
+      "epoch": 0.8436419515943142,
+      "grad_norm": 1.3765078783035278,
+      "learning_rate": 6.674398060854931e-05,
+      "loss": 1.6721,
+      "step": 2196
+    },
+    {
+      "epoch": 0.8459469842489435,
+      "grad_norm": 1.6013853549957275,
+      "learning_rate": 6.656427193593229e-05,
+      "loss": 1.6625,
+      "step": 2202
+    },
+    {
+      "epoch": 0.8482520169035728,
+      "grad_norm": 1.5348001718521118,
+      "learning_rate": 6.638432258249656e-05,
+      "loss": 1.7554,
+      "step": 2208
+    },
+    {
+      "epoch": 0.8505570495582021,
+      "grad_norm": 1.5851432085037231,
+      "learning_rate": 6.620413516292738e-05,
+      "loss": 1.742,
+      "step": 2214
+    },
+    {
+      "epoch": 0.8528620822128313,
+      "grad_norm": 1.3256301879882812,
+      "learning_rate": 6.602371229536916e-05,
+      "loss": 1.6646,
+      "step": 2220
+    },
+    {
+      "epoch": 0.8551671148674607,
+      "grad_norm": 1.3694243431091309,
+      "learning_rate": 6.584305660138734e-05,
+      "loss": 1.727,
+      "step": 2226
+    },
+    {
+      "epoch": 0.8574721475220899,
+      "grad_norm": 1.4219764471054077,
+      "learning_rate": 6.566217070593043e-05,
+      "loss": 1.6422,
+      "step": 2232
+    },
+    {
+      "epoch": 0.8597771801767192,
+      "grad_norm": 1.32774019241333,
+      "learning_rate": 6.548105723729175e-05,
+      "loss": 1.7207,
+      "step": 2238
+    },
+    {
+      "epoch": 0.8620822128313484,
+      "grad_norm": 1.6462959051132202,
+      "learning_rate": 6.529971882707127e-05,
+      "loss": 1.6939,
+      "step": 2244
+    },
+    {
+      "epoch": 0.8643872454859777,
+      "grad_norm": 1.4944287538528442,
+      "learning_rate": 6.511815811013745e-05,
+      "loss": 1.6897,
+      "step": 2250
+    },
+    {
+      "epoch": 0.866692278140607,
+      "grad_norm": 1.4976565837860107,
+      "learning_rate": 6.493637772458879e-05,
+      "loss": 1.7239,
+      "step": 2256
+    },
+    {
+      "epoch": 0.8689973107952362,
+      "grad_norm": 1.4712796211242676,
+      "learning_rate": 6.475438031171574e-05,
+      "loss": 1.8389,
+      "step": 2262
+    },
+    {
+      "epoch": 0.8713023434498656,
+      "grad_norm": 1.4492267370224,
+      "learning_rate": 6.457216851596207e-05,
+      "loss": 1.7952,
+      "step": 2268
+    },
+    {
+      "epoch": 0.8736073761044948,
+      "grad_norm": 1.330140471458435,
+      "learning_rate": 6.438974498488658e-05,
+      "loss": 1.7836,
+      "step": 2274
+    },
+    {
+      "epoch": 0.8759124087591241,
+      "grad_norm": 1.5496619939804077,
+      "learning_rate": 6.420711236912466e-05,
+      "loss": 1.8134,
+      "step": 2280
+    },
+    {
+      "epoch": 0.8782174414137534,
+      "grad_norm": 1.4317504167556763,
+      "learning_rate": 6.402427332234965e-05,
+      "loss": 1.713,
+      "step": 2286
+    },
+    {
+      "epoch": 0.8805224740683827,
+      "grad_norm": 1.664164662361145,
+      "learning_rate": 6.384123050123443e-05,
+      "loss": 1.7774,
+      "step": 2292
+    },
+    {
+      "epoch": 0.8828275067230119,
+      "grad_norm": 1.64762282371521,
+      "learning_rate": 6.36579865654127e-05,
+      "loss": 1.6297,
+      "step": 2298
+    },
+    {
+      "epoch": 0.8851325393776411,
+      "grad_norm": 1.3597866296768188,
+      "learning_rate": 6.347454417744038e-05,
+      "loss": 1.7333,
+      "step": 2304
+    },
+    {
+      "epoch": 0.8874375720322705,
+      "grad_norm": 1.4559557437896729,
+      "learning_rate": 6.329090600275696e-05,
+      "loss": 1.6322,
+      "step": 2310
+    },
+    {
+      "epoch": 0.8897426046868997,
+      "grad_norm": 1.6557890176773071,
+      "learning_rate": 6.310707470964668e-05,
+      "loss": 1.7589,
+      "step": 2316
+    },
+    {
+      "epoch": 0.892047637341529,
+      "grad_norm": 1.445709466934204,
+      "learning_rate": 6.292305296919986e-05,
+      "loss": 1.794,
+      "step": 2322
+    },
+    {
+      "epoch": 0.8943526699961583,
+      "grad_norm": 1.573336124420166,
+      "learning_rate": 6.273884345527402e-05,
+      "loss": 1.7086,
+      "step": 2328
+    },
+    {
+      "epoch": 0.8966577026507876,
+      "grad_norm": 1.8349642753601074,
+      "learning_rate": 6.255444884445506e-05,
+      "loss": 1.6558,
+      "step": 2334
+    },
+    {
+      "epoch": 0.8989627353054168,
+      "grad_norm": 1.4323902130126953,
+      "learning_rate": 6.236987181601833e-05,
+      "loss": 1.6671,
+      "step": 2340
+    },
+    {
+      "epoch": 0.9012677679600462,
+      "grad_norm": 1.2949434518814087,
+      "learning_rate": 6.21851150518898e-05,
+      "loss": 1.764,
+      "step": 2346
+    },
+    {
+      "epoch": 0.9035728006146754,
+      "grad_norm": 1.5487234592437744,
+      "learning_rate": 6.200018123660693e-05,
+      "loss": 1.6263,
+      "step": 2352
+    },
+    {
+      "epoch": 0.9058778332693046,
+      "grad_norm": 1.3451590538024902,
+      "learning_rate": 6.181507305727982e-05,
+      "loss": 1.6206,
+      "step": 2358
+    },
+    {
+      "epoch": 0.9081828659239339,
+      "grad_norm": 1.7791885137557983,
+      "learning_rate": 6.162979320355208e-05,
+      "loss": 1.7026,
+      "step": 2364
+    },
+    {
+      "epoch": 0.9104878985785632,
+      "grad_norm": 1.4890539646148682,
+      "learning_rate": 6.144434436756177e-05,
+      "loss": 1.7659,
+      "step": 2370
+    },
+    {
+      "epoch": 0.9127929312331925,
+      "grad_norm": 1.5732675790786743,
+      "learning_rate": 6.125872924390226e-05,
+      "loss": 1.5391,
+      "step": 2376
+    },
+    {
+      "epoch": 0.9150979638878217,
+      "grad_norm": 1.413071632385254,
+      "learning_rate": 6.107295052958314e-05,
+      "loss": 1.7246,
+      "step": 2382
+    },
+    {
+      "epoch": 0.917402996542451,
+      "grad_norm": 1.3524329662322998,
+      "learning_rate": 6.088701092399098e-05,
+      "loss": 1.8802,
+      "step": 2388
+    },
+    {
+      "epoch": 0.9197080291970803,
+      "grad_norm": 1.3901954889297485,
+      "learning_rate": 6.070091312885009e-05,
+      "loss": 1.6426,
+      "step": 2394
+    },
+    {
+      "epoch": 0.9220130618517096,
+      "grad_norm": 1.5605789422988892,
+      "learning_rate": 6.051465984818332e-05,
+      "loss": 1.6858,
+      "step": 2400
+    },
+    {
+      "epoch": 0.9243180945063388,
+      "grad_norm": 1.3955610990524292,
+      "learning_rate": 6.032825378827273e-05,
+      "loss": 1.7385,
+      "step": 2406
+    },
+    {
+      "epoch": 0.9266231271609681,
+      "grad_norm": 1.31680166721344,
+      "learning_rate": 6.0141697657620286e-05,
+      "loss": 1.8204,
+      "step": 2412
+    },
+    {
+      "epoch": 0.9289281598155974,
+      "grad_norm": 1.4082186222076416,
+      "learning_rate": 5.995499416690852e-05,
+      "loss": 1.8039,
+      "step": 2418
+    },
+    {
+      "epoch": 0.9312331924702266,
+      "grad_norm": 1.5661358833312988,
+      "learning_rate": 5.9768146028961094e-05,
+      "loss": 1.6775,
+      "step": 2424
+    },
+    {
+      "epoch": 0.933538225124856,
+      "grad_norm": 1.464074969291687,
+      "learning_rate": 5.958115595870341e-05,
+      "loss": 1.6794,
+      "step": 2430
+    },
+    {
+      "epoch": 0.9358432577794852,
+      "grad_norm": 1.6433953046798706,
+      "learning_rate": 5.9394026673123157e-05,
+      "loss": 1.7312,
+      "step": 2436
+    },
+    {
+      "epoch": 0.9381482904341145,
+      "grad_norm": 1.5434396266937256,
+      "learning_rate": 5.92067608912309e-05,
+      "loss": 1.6757,
+      "step": 2442
+    },
+    {
+      "epoch": 0.9404533230887437,
+      "grad_norm": 1.5374269485473633,
+      "learning_rate": 5.901936133402044e-05,
+      "loss": 1.6967,
+      "step": 2448
+    },
+    {
+      "epoch": 0.9427583557433731,
+      "grad_norm": 1.5485833883285522,
+      "learning_rate": 5.8831830724429384e-05,
+      "loss": 1.7169,
+      "step": 2454
+    },
+    {
+      "epoch": 0.9450633883980023,
+      "grad_norm": 1.637505054473877,
+      "learning_rate": 5.864417178729953e-05,
+      "loss": 1.7018,
+      "step": 2460
+    },
+    {
+      "epoch": 0.9473684210526315,
+      "grad_norm": 1.4926055669784546,
+      "learning_rate": 5.845638724933728e-05,
+      "loss": 1.6527,
+      "step": 2466
+    },
+    {
+      "epoch": 0.9496734537072609,
+      "grad_norm": 1.4670586585998535,
+      "learning_rate": 5.82684798390741e-05,
+      "loss": 1.6018,
+      "step": 2472
+    },
+    {
+      "epoch": 0.9519784863618901,
+      "grad_norm": 1.5422409772872925,
+      "learning_rate": 5.808045228682672e-05,
+      "loss": 1.7671,
+      "step": 2478
+    },
+    {
+      "epoch": 0.9542835190165194,
+      "grad_norm": 1.3220149278640747,
+      "learning_rate": 5.789230732465758e-05,
+      "loss": 1.6471,
+      "step": 2484
+    },
+    {
+      "epoch": 0.9565885516711486,
+      "grad_norm": 1.6373409032821655,
+      "learning_rate": 5.770404768633511e-05,
+      "loss": 1.7442,
+      "step": 2490
+    },
+    {
+      "epoch": 0.958893584325778,
+      "grad_norm": 1.361026406288147,
+      "learning_rate": 5.751567610729398e-05,
+      "loss": 1.7554,
+      "step": 2496
+    },
+    {
+      "epoch": 0.9611986169804072,
+      "grad_norm": 1.439842939376831,
+      "learning_rate": 5.7327195324595374e-05,
+      "loss": 1.5931,
+      "step": 2502
+    },
+    {
+      "epoch": 0.9635036496350365,
+      "grad_norm": 1.77671217918396,
+      "learning_rate": 5.7138608076887245e-05,
+      "loss": 1.6791,
+      "step": 2508
+    },
+    {
+      "epoch": 0.9658086822896658,
+      "grad_norm": 1.5075839757919312,
+      "learning_rate": 5.6949917104364456e-05,
+      "loss": 1.7447,
+      "step": 2514
+    },
+    {
+      "epoch": 0.968113714944295,
+      "grad_norm": 1.3645355701446533,
+      "learning_rate": 5.676112514872904e-05,
+      "loss": 1.5883,
+      "step": 2520
+    },
+    {
+      "epoch": 0.9704187475989243,
+      "grad_norm": 1.2660616636276245,
+      "learning_rate": 5.6572234953150305e-05,
+      "loss": 1.6906,
+      "step": 2526
+    },
+    {
+      "epoch": 0.9727237802535535,
+      "grad_norm": 1.470083475112915,
+      "learning_rate": 5.638324926222498e-05,
+      "loss": 1.6836,
+      "step": 2532
+    },
+    {
+      "epoch": 0.9750288129081829,
+      "grad_norm": 1.4210665225982666,
+      "learning_rate": 5.61941708219374e-05,
+      "loss": 1.8045,
+      "step": 2538
+    },
+    {
+      "epoch": 0.9773338455628121,
+      "grad_norm": 1.3398628234863281,
+      "learning_rate": 5.600500237961952e-05,
+      "loss": 1.6701,
+      "step": 2544
+    },
+    {
+      "epoch": 0.9796388782174414,
+      "grad_norm": 1.4541316032409668,
+      "learning_rate": 5.581574668391105e-05,
+      "loss": 1.6853,
+      "step": 2550
+    },
+    {
+      "epoch": 0.9819439108720707,
+      "grad_norm": 1.593572974205017,
+      "learning_rate": 5.5626406484719506e-05,
+      "loss": 1.6415,
+      "step": 2556
+    },
+    {
+      "epoch": 0.9842489435267,
+      "grad_norm": 1.6398078203201294,
+      "learning_rate": 5.5436984533180224e-05,
+      "loss": 1.6964,
+      "step": 2562
+    },
+    {
+      "epoch": 0.9865539761813292,
+      "grad_norm": 1.362488865852356,
+      "learning_rate": 5.524748358161644e-05,
+      "loss": 1.6616,
+      "step": 2568
+    },
+    {
+      "epoch": 0.9888590088359586,
+      "grad_norm": 1.3850921392440796,
+      "learning_rate": 5.505790638349928e-05,
+      "loss": 1.6569,
+      "step": 2574
+    },
+    {
+      "epoch": 0.9911640414905878,
+      "grad_norm": 1.6175508499145508,
+      "learning_rate": 5.48682556934077e-05,
+      "loss": 1.615,
+      "step": 2580
+    },
+    {
+      "epoch": 0.993469074145217,
+      "grad_norm": 1.4643458127975464,
+      "learning_rate": 5.467853426698853e-05,
+      "loss": 1.6656,
+      "step": 2586
+    },
+    {
+      "epoch": 0.9957741067998463,
+      "grad_norm": 1.4307057857513428,
+      "learning_rate": 5.448874486091637e-05,
+      "loss": 1.7121,
+      "step": 2592
+    },
+    {
+      "epoch": 0.9980791394544756,
+      "grad_norm": 1.4324171543121338,
+      "learning_rate": 5.429889023285366e-05,
+      "loss": 1.7007,
+      "step": 2598
+    }
+  ],
+  "logging_steps": 6,
+  "max_steps": 5206,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 2603,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.404676263684342e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2603/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d28e17ec4d61a1ab92dd2c3862d7a511ac560484e0933defc7b82fda4cf072cd
+size 5432

checkpoint-5206/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/OLMo-2-1124-7B-bho_Deva-coli-mean/",
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100384
+}

checkpoint-5206/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 100257,
+  "pad_token_id": 100277,
+  "transformers_version": "4.48.0.dev0"
+}

checkpoint-5206/model-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b5ebd633e26db41254f87a1b5ada88994420c8ea4d31799c34a6e39beba4f3
+size 4950070472

checkpoint-5206/model-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0b6a357d03676c4986588aaf4dc96afc3f44a0498514961d2b2cfa1940655e
+size 4991622816

checkpoint-5206/model-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1038a7d346a681a8fb6949e054c2246f3f3c80e826855c1cec6af1155cc607d
+size 4924546896

checkpoint-5206/model-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c0a21146ace02d36928787ae7f34be868f7c81b894dea98ec7b0e5a3ea01b2
+size 4857404928

checkpoint-5206/model-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f82a3014ccfc0f67208f63785ae4bc71f09aba86db48bf6139d18178abbfed
+size 4857404928

checkpoint-5206/model-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:271aef3d5032da51b6b587eeb5d199ab6065108544ac2a7181f8151bfe44312b
+size 4614509080

checkpoint-5206/model.safetensors.index.json

@@ -0,0 +1,362 @@
+{
+  "metadata": {
+    "total_size": 29195517952
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

checkpoint-5206/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e4991182c40ce98b0f04b1877fb3f9b424be51c0b6bef03910cad356d980c3c
+size 13055369358

checkpoint-5206/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff264f99d31b522cc7e2a4eac9d38606d0c58a34c0adc74d71e0ca8b371dc36
+size 14244

checkpoint-5206/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:717eb6a9fd9268e71bac43d97c38dcc58e38e22204a88526407fa88f5e8c885f
+size 1064

checkpoint-5206/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d28e17ec4d61a1ab92dd2c3862d7a511ac560484e0933defc7b82fda4cf072cd
+size 5432

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/OLMo-2-1124-7B-bho_Deva-coli-mean/",
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100384
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 100257,
+  "pad_token_id": 100277,
+  "transformers_version": "4.48.0.dev0"
+}

model-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b5ebd633e26db41254f87a1b5ada88994420c8ea4d31799c34a6e39beba4f3
+size 4950070472

model-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0b6a357d03676c4986588aaf4dc96afc3f44a0498514961d2b2cfa1940655e
+size 4991622816

model-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1038a7d346a681a8fb6949e054c2246f3f3c80e826855c1cec6af1155cc607d
+size 4924546896

model-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c0a21146ace02d36928787ae7f34be868f7c81b894dea98ec7b0e5a3ea01b2
+size 4857404928

model-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f82a3014ccfc0f67208f63785ae4bc71f09aba86db48bf6139d18178abbfed
+size 4857404928

model-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:271aef3d5032da51b6b587eeb5d199ab6065108544ac2a7181f8151bfe44312b
+size 4614509080

model.safetensors.index.json

@@ -0,0 +1,362 @@
+{
+  "metadata": {
+    "total_size": 29195517952
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,189 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d28e17ec4d61a1ab92dd2c3862d7a511ac560484e0933defc7b82fda4cf072cd
+size 5432