Upload folder using huggingface_hub

checkpoint-1054/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/OLMo-2-1124-7B-mni_Beng-coli-rand/",
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100384
+}

checkpoint-1054/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 100257,
+  "pad_token_id": 100277,
+  "transformers_version": "4.48.0.dev0"
+}

checkpoint-1054/model-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d1480335fcfd79184fe631e1b0aa77a59ac348fa5ac329223dcdf46a1bd81bd
+size 4950070472

checkpoint-1054/model-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0b6a357d03676c4986588aaf4dc96afc3f44a0498514961d2b2cfa1940655e
+size 4991622816

checkpoint-1054/model-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1038a7d346a681a8fb6949e054c2246f3f3c80e826855c1cec6af1155cc607d
+size 4924546896

checkpoint-1054/model-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c0a21146ace02d36928787ae7f34be868f7c81b894dea98ec7b0e5a3ea01b2
+size 4857404928

checkpoint-1054/model-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f82a3014ccfc0f67208f63785ae4bc71f09aba86db48bf6139d18178abbfed
+size 4857404928

checkpoint-1054/model-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a028ac98f31b7584432073229033962115043c4b055b0381c6d75b53761827f3
+size 4614509080

checkpoint-1054/model.safetensors.index.json

@@ -0,0 +1,362 @@
+{
+  "metadata": {
+    "total_size": 29195517952
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

checkpoint-1054/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b455ac8d50a9d8404fdd36ecd65632a63e8ad8fa71c122823faade099d5c4f36
+size 13055369358

checkpoint-1054/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff264f99d31b522cc7e2a4eac9d38606d0c58a34c0adc74d71e0ca8b371dc36
+size 14244

checkpoint-1054/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc23b7ea1627a8ce887ca6d1916045e948746b712746e84e85c5585c1ef54898
+size 1064

checkpoint-1054/trainer_state.json

@@ -0,0 +1,3722 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 1054,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003795066413662239,
+      "grad_norm": 98.7540512084961,
+      "learning_rate": 3.7735849056603773e-06,
+      "loss": 14.2228,
+      "step": 2
+    },
+    {
+      "epoch": 0.007590132827324478,
+      "grad_norm": 99.5528564453125,
+      "learning_rate": 7.547169811320755e-06,
+      "loss": 13.3495,
+      "step": 4
+    },
+    {
+      "epoch": 0.011385199240986717,
+      "grad_norm": 83.69811248779297,
+      "learning_rate": 1.1320754716981132e-05,
+      "loss": 12.5304,
+      "step": 6
+    },
+    {
+      "epoch": 0.015180265654648957,
+      "grad_norm": 47.37470626831055,
+      "learning_rate": 1.509433962264151e-05,
+      "loss": 9.4493,
+      "step": 8
+    },
+    {
+      "epoch": 0.018975332068311195,
+      "grad_norm": 31.21275520324707,
+      "learning_rate": 1.8867924528301888e-05,
+      "loss": 8.0006,
+      "step": 10
+    },
+    {
+      "epoch": 0.022770398481973434,
+      "grad_norm": 15.287549018859863,
+      "learning_rate": 2.2641509433962265e-05,
+      "loss": 6.4803,
+      "step": 12
+    },
+    {
+      "epoch": 0.026565464895635674,
+      "grad_norm": 50.968772888183594,
+      "learning_rate": 2.641509433962264e-05,
+      "loss": 6.7702,
+      "step": 14
+    },
+    {
+      "epoch": 0.030360531309297913,
+      "grad_norm": 28.34760856628418,
+      "learning_rate": 3.018867924528302e-05,
+      "loss": 6.2775,
+      "step": 16
+    },
+    {
+      "epoch": 0.03415559772296015,
+      "grad_norm": 24.7742862701416,
+      "learning_rate": 3.39622641509434e-05,
+      "loss": 5.9268,
+      "step": 18
+    },
+    {
+      "epoch": 0.03795066413662239,
+      "grad_norm": 62.243553161621094,
+      "learning_rate": 3.7735849056603776e-05,
+      "loss": 5.4478,
+      "step": 20
+    },
+    {
+      "epoch": 0.04174573055028463,
+      "grad_norm": 20.099931716918945,
+      "learning_rate": 4.150943396226415e-05,
+      "loss": 5.2172,
+      "step": 22
+    },
+    {
+      "epoch": 0.04554079696394687,
+      "grad_norm": 14.098124504089355,
+      "learning_rate": 4.528301886792453e-05,
+      "loss": 4.9133,
+      "step": 24
+    },
+    {
+      "epoch": 0.04933586337760911,
+      "grad_norm": 7.112690448760986,
+      "learning_rate": 4.9056603773584906e-05,
+      "loss": 4.7982,
+      "step": 26
+    },
+    {
+      "epoch": 0.05313092979127135,
+      "grad_norm": 13.210295677185059,
+      "learning_rate": 5.283018867924528e-05,
+      "loss": 4.7099,
+      "step": 28
+    },
+    {
+      "epoch": 0.056925996204933584,
+      "grad_norm": 68.6553726196289,
+      "learning_rate": 5.660377358490566e-05,
+      "loss": 4.3044,
+      "step": 30
+    },
+    {
+      "epoch": 0.06072106261859583,
+      "grad_norm": 7.807187557220459,
+      "learning_rate": 6.037735849056604e-05,
+      "loss": 4.2902,
+      "step": 32
+    },
+    {
+      "epoch": 0.06451612903225806,
+      "grad_norm": 24.964153289794922,
+      "learning_rate": 6.415094339622641e-05,
+      "loss": 4.0632,
+      "step": 34
+    },
+    {
+      "epoch": 0.0683111954459203,
+      "grad_norm": 5.0253095626831055,
+      "learning_rate": 6.79245283018868e-05,
+      "loss": 4.0919,
+      "step": 36
+    },
+    {
+      "epoch": 0.07210626185958255,
+      "grad_norm": 10.098166465759277,
+      "learning_rate": 7.169811320754717e-05,
+      "loss": 3.9745,
+      "step": 38
+    },
+    {
+      "epoch": 0.07590132827324478,
+      "grad_norm": 6.226728439331055,
+      "learning_rate": 7.547169811320755e-05,
+      "loss": 4.0257,
+      "step": 40
+    },
+    {
+      "epoch": 0.07969639468690702,
+      "grad_norm": 9.49015998840332,
+      "learning_rate": 7.924528301886794e-05,
+      "loss": 3.6716,
+      "step": 42
+    },
+    {
+      "epoch": 0.08349146110056926,
+      "grad_norm": 8.414839744567871,
+      "learning_rate": 8.30188679245283e-05,
+      "loss": 3.8115,
+      "step": 44
+    },
+    {
+      "epoch": 0.0872865275142315,
+      "grad_norm": 7.90946102142334,
+      "learning_rate": 8.679245283018869e-05,
+      "loss": 3.8998,
+      "step": 46
+    },
+    {
+      "epoch": 0.09108159392789374,
+      "grad_norm": 6.811845302581787,
+      "learning_rate": 9.056603773584906e-05,
+      "loss": 3.72,
+      "step": 48
+    },
+    {
+      "epoch": 0.09487666034155598,
+      "grad_norm": 8.63953971862793,
+      "learning_rate": 9.433962264150944e-05,
+      "loss": 3.8366,
+      "step": 50
+    },
+    {
+      "epoch": 0.09867172675521822,
+      "grad_norm": 5.25341272354126,
+      "learning_rate": 9.811320754716981e-05,
+      "loss": 3.6955,
+      "step": 52
+    },
+    {
+      "epoch": 0.10246679316888045,
+      "grad_norm": 6.284265041351318,
+      "learning_rate": 9.999975375283309e-05,
+      "loss": 3.5963,
+      "step": 54
+    },
+    {
+      "epoch": 0.1062618595825427,
+      "grad_norm": 8.026937484741211,
+      "learning_rate": 9.999778379005078e-05,
+      "loss": 3.668,
+      "step": 56
+    },
+    {
+      "epoch": 0.11005692599620494,
+      "grad_norm": 9.422395706176758,
+      "learning_rate": 9.99938439421016e-05,
+      "loss": 3.6292,
+      "step": 58
+    },
+    {
+      "epoch": 0.11385199240986717,
+      "grad_norm": 3.755744695663452,
+      "learning_rate": 9.99879343642134e-05,
+      "loss": 3.5123,
+      "step": 60
+    },
+    {
+      "epoch": 0.11764705882352941,
+      "grad_norm": 8.59239387512207,
+      "learning_rate": 9.99800552892203e-05,
+      "loss": 3.5657,
+      "step": 62
+    },
+    {
+      "epoch": 0.12144212523719165,
+      "grad_norm": 6.495412826538086,
+      "learning_rate": 9.997020702755353e-05,
+      "loss": 3.4943,
+      "step": 64
+    },
+    {
+      "epoch": 0.1252371916508539,
+      "grad_norm": 4.058749675750732,
+      "learning_rate": 9.995838996722914e-05,
+      "loss": 3.3784,
+      "step": 66
+    },
+    {
+      "epoch": 0.12903225806451613,
+      "grad_norm": 5.1243696212768555,
+      "learning_rate": 9.994460457383284e-05,
+      "loss": 3.5299,
+      "step": 68
+    },
+    {
+      "epoch": 0.13282732447817835,
+      "grad_norm": 3.1002564430236816,
+      "learning_rate": 9.992885139050154e-05,
+      "loss": 3.233,
+      "step": 70
+    },
+    {
+      "epoch": 0.1366223908918406,
+      "grad_norm": 9.574880599975586,
+      "learning_rate": 9.991113103790198e-05,
+      "loss": 3.3179,
+      "step": 72
+    },
+    {
+      "epoch": 0.14041745730550284,
+      "grad_norm": 4.202469825744629,
+      "learning_rate": 9.98914442142063e-05,
+      "loss": 3.2057,
+      "step": 74
+    },
+    {
+      "epoch": 0.1442125237191651,
+      "grad_norm": 9.224685668945312,
+      "learning_rate": 9.986979169506453e-05,
+      "loss": 3.2404,
+      "step": 76
+    },
+    {
+      "epoch": 0.14800759013282733,
+      "grad_norm": 6.027818202972412,
+      "learning_rate": 9.9846174333574e-05,
+      "loss": 3.3198,
+      "step": 78
+    },
+    {
+      "epoch": 0.15180265654648956,
+      "grad_norm": 5.427716255187988,
+      "learning_rate": 9.982059306024577e-05,
+      "loss": 3.2348,
+      "step": 80
+    },
+    {
+      "epoch": 0.1555977229601518,
+      "grad_norm": 3.3449645042419434,
+      "learning_rate": 9.979304888296792e-05,
+      "loss": 3.1103,
+      "step": 82
+    },
+    {
+      "epoch": 0.15939278937381404,
+      "grad_norm": 3.7018656730651855,
+      "learning_rate": 9.976354288696588e-05,
+      "loss": 3.0045,
+      "step": 84
+    },
+    {
+      "epoch": 0.16318785578747627,
+      "grad_norm": 3.1451151371002197,
+      "learning_rate": 9.973207623475965e-05,
+      "loss": 3.2071,
+      "step": 86
+    },
+    {
+      "epoch": 0.16698292220113853,
+      "grad_norm": 3.133887529373169,
+      "learning_rate": 9.9698650166118e-05,
+      "loss": 3.0677,
+      "step": 88
+    },
+    {
+      "epoch": 0.17077798861480076,
+      "grad_norm": 5.542338848114014,
+      "learning_rate": 9.966326599800967e-05,
+      "loss": 3.0122,
+      "step": 90
+    },
+    {
+      "epoch": 0.174573055028463,
+      "grad_norm": 3.292200803756714,
+      "learning_rate": 9.962592512455138e-05,
+      "loss": 2.9215,
+      "step": 92
+    },
+    {
+      "epoch": 0.17836812144212524,
+      "grad_norm": 3.4283339977264404,
+      "learning_rate": 9.958662901695303e-05,
+      "loss": 2.9141,
+      "step": 94
+    },
+    {
+      "epoch": 0.18216318785578747,
+      "grad_norm": 3.9418880939483643,
+      "learning_rate": 9.954537922345961e-05,
+      "loss": 2.7547,
+      "step": 96
+    },
+    {
+      "epoch": 0.1859582542694497,
+      "grad_norm": 3.3023762702941895,
+      "learning_rate": 9.950217736929029e-05,
+      "loss": 2.8594,
+      "step": 98
+    },
+    {
+      "epoch": 0.18975332068311196,
+      "grad_norm": 3.1366522312164307,
+      "learning_rate": 9.945702515657434e-05,
+      "loss": 2.8524,
+      "step": 100
+    },
+    {
+      "epoch": 0.1935483870967742,
+      "grad_norm": 2.613105535507202,
+      "learning_rate": 9.940992436428409e-05,
+      "loss": 2.8663,
+      "step": 102
+    },
+    {
+      "epoch": 0.19734345351043645,
+      "grad_norm": 5.562861442565918,
+      "learning_rate": 9.936087684816486e-05,
+      "loss": 3.1047,
+      "step": 104
+    },
+    {
+      "epoch": 0.20113851992409867,
+      "grad_norm": 4.076806545257568,
+      "learning_rate": 9.930988454066177e-05,
+      "loss": 2.7816,
+      "step": 106
+    },
+    {
+      "epoch": 0.2049335863377609,
+      "grad_norm": 2.7189831733703613,
+      "learning_rate": 9.92569494508437e-05,
+      "loss": 2.8001,
+      "step": 108
+    },
+    {
+      "epoch": 0.20872865275142316,
+      "grad_norm": 4.94251012802124,
+      "learning_rate": 9.920207366432402e-05,
+      "loss": 2.7223,
+      "step": 110
+    },
+    {
+      "epoch": 0.2125237191650854,
+      "grad_norm": 2.9818546772003174,
+      "learning_rate": 9.914525934317855e-05,
+      "loss": 2.8046,
+      "step": 112
+    },
+    {
+      "epoch": 0.21631878557874762,
+      "grad_norm": 3.87678599357605,
+      "learning_rate": 9.908650872586029e-05,
+      "loss": 2.7476,
+      "step": 114
+    },
+    {
+      "epoch": 0.22011385199240988,
+      "grad_norm": 2.9942760467529297,
+      "learning_rate": 9.90258241271112e-05,
+      "loss": 2.6728,
+      "step": 116
+    },
+    {
+      "epoch": 0.2239089184060721,
+      "grad_norm": 4.912055969238281,
+      "learning_rate": 9.896320793787106e-05,
+      "loss": 2.6644,
+      "step": 118
+    },
+    {
+      "epoch": 0.22770398481973433,
+      "grad_norm": 3.1557791233062744,
+      "learning_rate": 9.889866262518331e-05,
+      "loss": 2.7656,
+      "step": 120
+    },
+    {
+      "epoch": 0.2314990512333966,
+      "grad_norm": 8.984029769897461,
+      "learning_rate": 9.883219073209772e-05,
+      "loss": 2.7668,
+      "step": 122
+    },
+    {
+      "epoch": 0.23529411764705882,
+      "grad_norm": 6.998969078063965,
+      "learning_rate": 9.876379487757034e-05,
+      "loss": 2.7245,
+      "step": 124
+    },
+    {
+      "epoch": 0.23908918406072105,
+      "grad_norm": 2.730355978012085,
+      "learning_rate": 9.869347775636015e-05,
+      "loss": 2.7342,
+      "step": 126
+    },
+    {
+      "epoch": 0.2428842504743833,
+      "grad_norm": 8.605497360229492,
+      "learning_rate": 9.862124213892304e-05,
+      "loss": 2.776,
+      "step": 128
+    },
+    {
+      "epoch": 0.24667931688804554,
+      "grad_norm": 5.028979301452637,
+      "learning_rate": 9.85470908713026e-05,
+      "loss": 2.722,
+      "step": 130
+    },
+    {
+      "epoch": 0.2504743833017078,
+      "grad_norm": 3.3529725074768066,
+      "learning_rate": 9.847102687501797e-05,
+      "loss": 2.7299,
+      "step": 132
+    },
+    {
+      "epoch": 0.25426944971537,
+      "grad_norm": 4.99077844619751,
+      "learning_rate": 9.839305314694873e-05,
+      "loss": 2.7569,
+      "step": 134
+    },
+    {
+      "epoch": 0.25806451612903225,
+      "grad_norm": 5.339693546295166,
+      "learning_rate": 9.831317275921685e-05,
+      "loss": 2.579,
+      "step": 136
+    },
+    {
+      "epoch": 0.2618595825426945,
+      "grad_norm": 5.73229455947876,
+      "learning_rate": 9.823138885906566e-05,
+      "loss": 2.5465,
+      "step": 138
+    },
+    {
+      "epoch": 0.2656546489563567,
+      "grad_norm": 2.466779947280884,
+      "learning_rate": 9.814770466873585e-05,
+      "loss": 2.7257,
+      "step": 140
+    },
+    {
+      "epoch": 0.269449715370019,
+      "grad_norm": 4.790260314941406,
+      "learning_rate": 9.806212348533838e-05,
+      "loss": 2.545,
+      "step": 142
+    },
+    {
+      "epoch": 0.2732447817836812,
+      "grad_norm": 4.156448841094971,
+      "learning_rate": 9.797464868072488e-05,
+      "loss": 2.6578,
+      "step": 144
+    },
+    {
+      "epoch": 0.27703984819734345,
+      "grad_norm": 2.7191035747528076,
+      "learning_rate": 9.788528370135443e-05,
+      "loss": 2.5779,
+      "step": 146
+    },
+    {
+      "epoch": 0.2808349146110057,
+      "grad_norm": 2.6859488487243652,
+      "learning_rate": 9.77940320681581e-05,
+      "loss": 2.5819,
+      "step": 148
+    },
+    {
+      "epoch": 0.2846299810246679,
+      "grad_norm": 3.9940454959869385,
+      "learning_rate": 9.77008973764e-05,
+      "loss": 2.417,
+      "step": 150
+    },
+    {
+      "epoch": 0.2884250474383302,
+      "grad_norm": 2.97194504737854,
+      "learning_rate": 9.760588329553571e-05,
+      "loss": 2.4018,
+      "step": 152
+    },
+    {
+      "epoch": 0.2922201138519924,
+      "grad_norm": 4.698624134063721,
+      "learning_rate": 9.750899356906775e-05,
+      "loss": 2.5237,
+      "step": 154
+    },
+    {
+      "epoch": 0.29601518026565465,
+      "grad_norm": 3.8055150508880615,
+      "learning_rate": 9.741023201439803e-05,
+      "loss": 2.4408,
+      "step": 156
+    },
+    {
+      "epoch": 0.2998102466793169,
+      "grad_norm": 3.4264442920684814,
+      "learning_rate": 9.730960252267743e-05,
+      "loss": 2.5248,
+      "step": 158
+    },
+    {
+      "epoch": 0.3036053130929791,
+      "grad_norm": 2.7180609703063965,
+      "learning_rate": 9.720710905865256e-05,
+      "loss": 2.4598,
+      "step": 160
+    },
+    {
+      "epoch": 0.30740037950664134,
+      "grad_norm": 4.305523872375488,
+      "learning_rate": 9.710275566050951e-05,
+      "loss": 2.5028,
+      "step": 162
+    },
+    {
+      "epoch": 0.3111954459203036,
+      "grad_norm": 2.756655693054199,
+      "learning_rate": 9.699654643971472e-05,
+      "loss": 2.5315,
+      "step": 164
+    },
+    {
+      "epoch": 0.31499051233396586,
+      "grad_norm": 2.449230432510376,
+      "learning_rate": 9.688848558085306e-05,
+      "loss": 2.3747,
+      "step": 166
+    },
+    {
+      "epoch": 0.3187855787476281,
+      "grad_norm": 3.7404186725616455,
+      "learning_rate": 9.677857734146289e-05,
+      "loss": 2.4413,
+      "step": 168
+    },
+    {
+      "epoch": 0.3225806451612903,
+      "grad_norm": 2.4831082820892334,
+      "learning_rate": 9.666682605186835e-05,
+      "loss": 2.5693,
+      "step": 170
+    },
+    {
+      "epoch": 0.32637571157495254,
+      "grad_norm": 3.9603495597839355,
+      "learning_rate": 9.655323611500875e-05,
+      "loss": 2.603,
+      "step": 172
+    },
+    {
+      "epoch": 0.3301707779886148,
+      "grad_norm": 2.5853846073150635,
+      "learning_rate": 9.643781200626511e-05,
+      "loss": 2.3858,
+      "step": 174
+    },
+    {
+      "epoch": 0.33396584440227706,
+      "grad_norm": 2.5267770290374756,
+      "learning_rate": 9.632055827328382e-05,
+      "loss": 2.4411,
+      "step": 176
+    },
+    {
+      "epoch": 0.3377609108159393,
+      "grad_norm": 2.9036340713500977,
+      "learning_rate": 9.620147953579737e-05,
+      "loss": 2.4533,
+      "step": 178
+    },
+    {
+      "epoch": 0.3415559772296015,
+      "grad_norm": 2.319636344909668,
+      "learning_rate": 9.608058048544251e-05,
+      "loss": 2.4877,
+      "step": 180
+    },
+    {
+      "epoch": 0.34535104364326374,
+      "grad_norm": 3.325531244277954,
+      "learning_rate": 9.595786588557532e-05,
+      "loss": 2.4027,
+      "step": 182
+    },
+    {
+      "epoch": 0.349146110056926,
+      "grad_norm": 11.142248153686523,
+      "learning_rate": 9.583334057108346e-05,
+      "loss": 2.3184,
+      "step": 184
+    },
+    {
+      "epoch": 0.35294117647058826,
+      "grad_norm": 2.423501968383789,
+      "learning_rate": 9.570700944819584e-05,
+      "loss": 2.5982,
+      "step": 186
+    },
+    {
+      "epoch": 0.3567362428842505,
+      "grad_norm": 2.4559223651885986,
+      "learning_rate": 9.557887749428913e-05,
+      "loss": 2.4276,
+      "step": 188
+    },
+    {
+      "epoch": 0.3605313092979127,
+      "grad_norm": 2.703495502471924,
+      "learning_rate": 9.544894975769186e-05,
+      "loss": 2.3763,
+      "step": 190
+    },
+    {
+      "epoch": 0.36432637571157495,
+      "grad_norm": 1.9051834344863892,
+      "learning_rate": 9.531723135748529e-05,
+      "loss": 2.3381,
+      "step": 192
+    },
+    {
+      "epoch": 0.3681214421252372,
+      "grad_norm": 2.33522629737854,
+      "learning_rate": 9.518372748330194e-05,
+      "loss": 2.3207,
+      "step": 194
+    },
+    {
+      "epoch": 0.3719165085388994,
+      "grad_norm": 2.3043212890625,
+      "learning_rate": 9.504844339512095e-05,
+      "loss": 2.2691,
+      "step": 196
+    },
+    {
+      "epoch": 0.3757115749525617,
+      "grad_norm": 2.3060500621795654,
+      "learning_rate": 9.4911384423061e-05,
+      "loss": 2.4741,
+      "step": 198
+    },
+    {
+      "epoch": 0.3795066413662239,
+      "grad_norm": 2.431614398956299,
+      "learning_rate": 9.477255596717012e-05,
+      "loss": 2.3706,
+      "step": 200
+    },
+    {
+      "epoch": 0.38330170777988615,
+      "grad_norm": 2.571744918823242,
+      "learning_rate": 9.463196349721308e-05,
+      "loss": 2.4345,
+      "step": 202
+    },
+    {
+      "epoch": 0.3870967741935484,
+      "grad_norm": 2.950385332107544,
+      "learning_rate": 9.448961255245584e-05,
+      "loss": 2.0978,
+      "step": 204
+    },
+    {
+      "epoch": 0.3908918406072106,
+      "grad_norm": 6.460769176483154,
+      "learning_rate": 9.434550874144728e-05,
+      "loss": 2.2981,
+      "step": 206
+    },
+    {
+      "epoch": 0.3946869070208729,
+      "grad_norm": 2.294574022293091,
+      "learning_rate": 9.419965774179824e-05,
+      "loss": 2.3483,
+      "step": 208
+    },
+    {
+      "epoch": 0.3984819734345351,
+      "grad_norm": 2.530029296875,
+      "learning_rate": 9.405206529995785e-05,
+      "loss": 2.4667,
+      "step": 210
+    },
+    {
+      "epoch": 0.40227703984819735,
+      "grad_norm": 2.0644922256469727,
+      "learning_rate": 9.39027372309871e-05,
+      "loss": 2.2927,
+      "step": 212
+    },
+    {
+      "epoch": 0.4060721062618596,
+      "grad_norm": 2.369694948196411,
+      "learning_rate": 9.375167941832973e-05,
+      "loss": 2.4687,
+      "step": 214
+    },
+    {
+      "epoch": 0.4098671726755218,
+      "grad_norm": 2.3800711631774902,
+      "learning_rate": 9.359889781358042e-05,
+      "loss": 2.3473,
+      "step": 216
+    },
+    {
+      "epoch": 0.41366223908918404,
+      "grad_norm": 2.2723309993743896,
+      "learning_rate": 9.344439843625034e-05,
+      "loss": 2.4378,
+      "step": 218
+    },
+    {
+      "epoch": 0.4174573055028463,
+      "grad_norm": 2.2413368225097656,
+      "learning_rate": 9.32881873735299e-05,
+      "loss": 2.391,
+      "step": 220
+    },
+    {
+      "epoch": 0.42125237191650855,
+      "grad_norm": 2.273754596710205,
+      "learning_rate": 9.313027078004903e-05,
+      "loss": 2.2899,
+      "step": 222
+    },
+    {
+      "epoch": 0.4250474383301708,
+      "grad_norm": 2.2374515533447266,
+      "learning_rate": 9.297065487763462e-05,
+      "loss": 2.3854,
+      "step": 224
+    },
+    {
+      "epoch": 0.428842504743833,
+      "grad_norm": 1.9647648334503174,
+      "learning_rate": 9.280934595506536e-05,
+      "loss": 2.1885,
+      "step": 226
+    },
+    {
+      "epoch": 0.43263757115749524,
+      "grad_norm": 2.4773480892181396,
+      "learning_rate": 9.264635036782405e-05,
+      "loss": 2.1969,
+      "step": 228
+    },
+    {
+      "epoch": 0.4364326375711575,
+      "grad_norm": 4.396137237548828,
+      "learning_rate": 9.248167453784711e-05,
+      "loss": 2.2705,
+      "step": 230
+    },
+    {
+      "epoch": 0.44022770398481975,
+      "grad_norm": 2.6246371269226074,
+      "learning_rate": 9.231532495327165e-05,
+      "loss": 2.3367,
+      "step": 232
+    },
+    {
+      "epoch": 0.444022770398482,
+      "grad_norm": 2.36479115486145,
+      "learning_rate": 9.21473081681797e-05,
+      "loss": 2.4254,
+      "step": 234
+    },
+    {
+      "epoch": 0.4478178368121442,
+      "grad_norm": 2.2640645503997803,
+      "learning_rate": 9.197763080234019e-05,
+      "loss": 2.198,
+      "step": 236
+    },
+    {
+      "epoch": 0.45161290322580644,
+      "grad_norm": 2.6508326530456543,
+      "learning_rate": 9.180629954094792e-05,
+      "loss": 2.253,
+      "step": 238
+    },
+    {
+      "epoch": 0.45540796963946867,
+      "grad_norm": 3.5617713928222656,
+      "learning_rate": 9.163332113436032e-05,
+      "loss": 2.1892,
+      "step": 240
+    },
+    {
+      "epoch": 0.45920303605313095,
+      "grad_norm": 2.4162395000457764,
+      "learning_rate": 9.145870239783142e-05,
+      "loss": 2.3496,
+      "step": 242
+    },
+    {
+      "epoch": 0.4629981024667932,
+      "grad_norm": 3.857107162475586,
+      "learning_rate": 9.128245021124334e-05,
+      "loss": 2.364,
+      "step": 244
+    },
+    {
+      "epoch": 0.4667931688804554,
+      "grad_norm": 2.012956142425537,
+      "learning_rate": 9.110457151883523e-05,
+      "loss": 2.2571,
+      "step": 246
+    },
+    {
+      "epoch": 0.47058823529411764,
+      "grad_norm": 1.9764139652252197,
+      "learning_rate": 9.092507332892968e-05,
+      "loss": 2.2254,
+      "step": 248
+    },
+    {
+      "epoch": 0.47438330170777987,
+      "grad_norm": 1.8924527168273926,
+      "learning_rate": 9.07439627136566e-05,
+      "loss": 2.1948,
+      "step": 250
+    },
+    {
+      "epoch": 0.4781783681214421,
+      "grad_norm": 2.961359739303589,
+      "learning_rate": 9.056124680867457e-05,
+      "loss": 2.1587,
+      "step": 252
+    },
+    {
+      "epoch": 0.4819734345351044,
+      "grad_norm": 2.010711193084717,
+      "learning_rate": 9.037693281288969e-05,
+      "loss": 2.3077,
+      "step": 254
+    },
+    {
+      "epoch": 0.4857685009487666,
+      "grad_norm": 2.0528528690338135,
+      "learning_rate": 9.019102798817197e-05,
+      "loss": 2.2348,
+      "step": 256
+    },
+    {
+      "epoch": 0.48956356736242884,
+      "grad_norm": 3.3071796894073486,
+      "learning_rate": 9.000353965906917e-05,
+      "loss": 2.3238,
+      "step": 258
+    },
+    {
+      "epoch": 0.49335863377609107,
+      "grad_norm": 2.2678682804107666,
+      "learning_rate": 8.981447521251831e-05,
+      "loss": 2.2368,
+      "step": 260
+    },
+    {
+      "epoch": 0.4971537001897533,
+      "grad_norm": 2.4855003356933594,
+      "learning_rate": 8.962384209755452e-05,
+      "loss": 2.0563,
+      "step": 262
+    },
+    {
+      "epoch": 0.5009487666034156,
+      "grad_norm": 2.1087934970855713,
+      "learning_rate": 8.943164782501765e-05,
+      "loss": 2.2727,
+      "step": 264
+    },
+    {
+      "epoch": 0.5047438330170778,
+      "grad_norm": 2.3885719776153564,
+      "learning_rate": 8.923789996725624e-05,
+      "loss": 2.2086,
+      "step": 266
+    },
+    {
+      "epoch": 0.50853889943074,
+      "grad_norm": 3.1673200130462646,
+      "learning_rate": 8.904260615782927e-05,
+      "loss": 2.1385,
+      "step": 268
+    },
+    {
+      "epoch": 0.5123339658444023,
+      "grad_norm": 2.06791353225708,
+      "learning_rate": 8.884577409120535e-05,
+      "loss": 2.2336,
+      "step": 270
+    },
+    {
+      "epoch": 0.5161290322580645,
+      "grad_norm": 2.3750381469726562,
+      "learning_rate": 8.864741152245963e-05,
+      "loss": 2.1569,
+      "step": 272
+    },
+    {
+      "epoch": 0.5199240986717267,
+      "grad_norm": 2.1411375999450684,
+      "learning_rate": 8.84475262669681e-05,
+      "loss": 2.2936,
+      "step": 274
+    },
+    {
+      "epoch": 0.523719165085389,
+      "grad_norm": 2.0463004112243652,
+      "learning_rate": 8.824612620009987e-05,
+      "loss": 2.3689,
+      "step": 276
+    },
+    {
+      "epoch": 0.5275142314990512,
+      "grad_norm": 1.7346696853637695,
+      "learning_rate": 8.804321925690672e-05,
+      "loss": 2.2445,
+      "step": 278
+    },
+    {
+      "epoch": 0.5313092979127134,
+      "grad_norm": 2.2764406204223633,
+      "learning_rate": 8.783881343181055e-05,
+      "loss": 2.1715,
+      "step": 280
+    },
+    {
+      "epoch": 0.5351043643263758,
+      "grad_norm": 2.2156970500946045,
+      "learning_rate": 8.763291677828838e-05,
+      "loss": 2.0616,
+      "step": 282
+    },
+    {
+      "epoch": 0.538899430740038,
+      "grad_norm": 2.32037091255188,
+      "learning_rate": 8.742553740855506e-05,
+      "loss": 2.2687,
+      "step": 284
+    },
+    {
+      "epoch": 0.5426944971537002,
+      "grad_norm": 1.864304542541504,
+      "learning_rate": 8.721668349324364e-05,
+      "loss": 2.2962,
+      "step": 286
+    },
+    {
+      "epoch": 0.5464895635673624,
+      "grad_norm": 2.056326389312744,
+      "learning_rate": 8.700636326108342e-05,
+      "loss": 2.3313,
+      "step": 288
+    },
+    {
+      "epoch": 0.5502846299810247,
+      "grad_norm": 2.4540798664093018,
+      "learning_rate": 8.679458499857582e-05,
+      "loss": 2.2259,
+      "step": 290
+    },
+    {
+      "epoch": 0.5540796963946869,
+      "grad_norm": 1.975327968597412,
+      "learning_rate": 8.658135704966786e-05,
+      "loss": 2.1908,
+      "step": 292
+    },
+    {
+      "epoch": 0.5578747628083491,
+      "grad_norm": 1.679593801498413,
+      "learning_rate": 8.636668781542336e-05,
+      "loss": 2.1169,
+      "step": 294
+    },
+    {
+      "epoch": 0.5616698292220114,
+      "grad_norm": 1.7750844955444336,
+      "learning_rate": 8.615058575369202e-05,
+      "loss": 2.0965,
+      "step": 296
+    },
+    {
+      "epoch": 0.5654648956356736,
+      "grad_norm": 1.9540330171585083,
+      "learning_rate": 8.593305937877614e-05,
+      "loss": 2.1185,
+      "step": 298
+    },
+    {
+      "epoch": 0.5692599620493358,
+      "grad_norm": 3.0039901733398438,
+      "learning_rate": 8.571411726109519e-05,
+      "loss": 2.2415,
+      "step": 300
+    },
+    {
+      "epoch": 0.573055028462998,
+      "grad_norm": 2.0792276859283447,
+      "learning_rate": 8.549376802684812e-05,
+      "loss": 2.2909,
+      "step": 302
+    },
+    {
+      "epoch": 0.5768500948766604,
+      "grad_norm": 1.9235445261001587,
+      "learning_rate": 8.527202035767349e-05,
+      "loss": 2.2691,
+      "step": 304
+    },
+    {
+      "epoch": 0.5806451612903226,
+      "grad_norm": 1.9486608505249023,
+      "learning_rate": 8.504888299030747e-05,
+      "loss": 2.141,
+      "step": 306
+    },
+    {
+      "epoch": 0.5844402277039848,
+      "grad_norm": 1.959185242652893,
+      "learning_rate": 8.482436471623951e-05,
+      "loss": 2.1965,
+      "step": 308
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 5.812629699707031,
+      "learning_rate": 8.459847438136605e-05,
+      "loss": 2.012,
+      "step": 310
+    },
+    {
+      "epoch": 0.5920303605313093,
+      "grad_norm": 1.9838637113571167,
+      "learning_rate": 8.437122088564198e-05,
+      "loss": 2.074,
+      "step": 312
+    },
+    {
+      "epoch": 0.5958254269449715,
+      "grad_norm": 1.9467284679412842,
+      "learning_rate": 8.414261318272996e-05,
+      "loss": 2.1882,
+      "step": 314
+    },
+    {
+      "epoch": 0.5996204933586338,
+      "grad_norm": 1.763068675994873,
+      "learning_rate": 8.391266027964771e-05,
+      "loss": 2.2197,
+      "step": 316
+    },
+    {
+      "epoch": 0.603415559772296,
+      "grad_norm": 2.123964309692383,
+      "learning_rate": 8.368137123641302e-05,
+      "loss": 2.1306,
+      "step": 318
+    },
+    {
+      "epoch": 0.6072106261859582,
+      "grad_norm": 1.9844577312469482,
+      "learning_rate": 8.344875516568695e-05,
+      "loss": 2.2215,
+      "step": 320
+    },
+    {
+      "epoch": 0.6110056925996205,
+      "grad_norm": 1.8439053297042847,
+      "learning_rate": 8.321482123241464e-05,
+      "loss": 2.0638,
+      "step": 322
+    },
+    {
+      "epoch": 0.6148007590132827,
+      "grad_norm": 1.7768611907958984,
+      "learning_rate": 8.297957865346437e-05,
+      "loss": 2.0954,
+      "step": 324
+    },
+    {
+      "epoch": 0.618595825426945,
+      "grad_norm": 2.3415918350219727,
+      "learning_rate": 8.274303669726426e-05,
+      "loss": 1.9216,
+      "step": 326
+    },
+    {
+      "epoch": 0.6223908918406073,
+      "grad_norm": 1.9908041954040527,
+      "learning_rate": 8.250520468343722e-05,
+      "loss": 2.1316,
+      "step": 328
+    },
+    {
+      "epoch": 0.6261859582542695,
+      "grad_norm": 1.989737868309021,
+      "learning_rate": 8.226609198243372e-05,
+      "loss": 2.1663,
+      "step": 330
+    },
+    {
+      "epoch": 0.6299810246679317,
+      "grad_norm": 2.194958209991455,
+      "learning_rate": 8.202570801516256e-05,
+      "loss": 2.0314,
+      "step": 332
+    },
+    {
+      "epoch": 0.6337760910815939,
+      "grad_norm": 1.9709926843643188,
+      "learning_rate": 8.178406225261981e-05,
+      "loss": 2.0792,
+      "step": 334
+    },
+    {
+      "epoch": 0.6375711574952562,
+      "grad_norm": 1.8470232486724854,
+      "learning_rate": 8.15411642155155e-05,
+      "loss": 2.0086,
+      "step": 336
+    },
+    {
+      "epoch": 0.6413662239089184,
+      "grad_norm": 1.961173415184021,
+      "learning_rate": 8.129702347389865e-05,
+      "loss": 2.1516,
+      "step": 338
+    },
+    {
+      "epoch": 0.6451612903225806,
+      "grad_norm": 1.9438867568969727,
+      "learning_rate": 8.105164964678009e-05,
+      "loss": 2.3398,
+      "step": 340
+    },
+    {
+      "epoch": 0.6489563567362429,
+      "grad_norm": 1.8040043115615845,
+      "learning_rate": 8.080505240175363e-05,
+      "loss": 2.1578,
+      "step": 342
+    },
+    {
+      "epoch": 0.6527514231499051,
+      "grad_norm": 2.0405383110046387,
+      "learning_rate": 8.055724145461495e-05,
+      "loss": 1.9948,
+      "step": 344
+    },
+    {
+      "epoch": 0.6565464895635673,
+      "grad_norm": 2.0209884643554688,
+      "learning_rate": 8.030822656897902e-05,
+      "loss": 2.1048,
+      "step": 346
+    },
+    {
+      "epoch": 0.6603415559772297,
+      "grad_norm": 1.9259743690490723,
+      "learning_rate": 8.005801755589532e-05,
+      "loss": 1.9457,
+      "step": 348
+    },
+    {
+      "epoch": 0.6641366223908919,
+      "grad_norm": 2.5305731296539307,
+      "learning_rate": 7.980662427346127e-05,
+      "loss": 1.8099,
+      "step": 350
+    },
+    {
+      "epoch": 0.6679316888045541,
+      "grad_norm": 2.0857901573181152,
+      "learning_rate": 7.955405662643384e-05,
+      "loss": 1.8904,
+      "step": 352
+    },
+    {
+      "epoch": 0.6717267552182163,
+      "grad_norm": 2.1652462482452393,
+      "learning_rate": 7.930032456583931e-05,
+      "loss": 2.0716,
+      "step": 354
+    },
+    {
+      "epoch": 0.6755218216318786,
+      "grad_norm": 2.2173564434051514,
+      "learning_rate": 7.904543808858127e-05,
+      "loss": 1.9861,
+      "step": 356
+    },
+    {
+      "epoch": 0.6793168880455408,
+      "grad_norm": 1.7087043523788452,
+      "learning_rate": 7.878940723704664e-05,
+      "loss": 1.9788,
+      "step": 358
+    },
+    {
+      "epoch": 0.683111954459203,
+      "grad_norm": 2.3170816898345947,
+      "learning_rate": 7.853224209871007e-05,
+      "loss": 2.0999,
+      "step": 360
+    },
+    {
+      "epoch": 0.6869070208728653,
+      "grad_norm": 1.8516716957092285,
+      "learning_rate": 7.82739528057365e-05,
+      "loss": 1.9476,
+      "step": 362
+    },
+    {
+      "epoch": 0.6907020872865275,
+      "grad_norm": 1.87705659866333,
+      "learning_rate": 7.801454953458193e-05,
+      "loss": 1.9741,
+      "step": 364
+    },
+    {
+      "epoch": 0.6944971537001897,
+      "grad_norm": 2.0102131366729736,
+      "learning_rate": 7.775404250559249e-05,
+      "loss": 2.2415,
+      "step": 366
+    },
+    {
+      "epoch": 0.698292220113852,
+      "grad_norm": 1.992819905281067,
+      "learning_rate": 7.749244198260175e-05,
+      "loss": 2.0201,
+      "step": 368
+    },
+    {
+      "epoch": 0.7020872865275142,
+      "grad_norm": 2.0950779914855957,
+      "learning_rate": 7.722975827252638e-05,
+      "loss": 2.2209,
+      "step": 370
+    },
+    {
+      "epoch": 0.7058823529411765,
+      "grad_norm": 7.29935884475708,
+      "learning_rate": 7.696600172495997e-05,
+      "loss": 2.0799,
+      "step": 372
+    },
+    {
+      "epoch": 0.7096774193548387,
+      "grad_norm": 1.9377464056015015,
+      "learning_rate": 7.670118273176534e-05,
+      "loss": 2.0681,
+      "step": 374
+    },
+    {
+      "epoch": 0.713472485768501,
+      "grad_norm": 2.465165853500366,
+      "learning_rate": 7.643531172666513e-05,
+      "loss": 2.0705,
+      "step": 376
+    },
+    {
+      "epoch": 0.7172675521821632,
+      "grad_norm": 1.7911758422851562,
+      "learning_rate": 7.616839918483061e-05,
+      "loss": 2.1271,
+      "step": 378
+    },
+    {
+      "epoch": 0.7210626185958254,
+      "grad_norm": 1.8003861904144287,
+      "learning_rate": 7.590045562246902e-05,
+      "loss": 2.0348,
+      "step": 380
+    },
+    {
+      "epoch": 0.7248576850094877,
+      "grad_norm": 1.9686284065246582,
+      "learning_rate": 7.563149159640929e-05,
+      "loss": 2.0137,
+      "step": 382
+    },
+    {
+      "epoch": 0.7286527514231499,
+      "grad_norm": 1.6355502605438232,
+      "learning_rate": 7.5361517703686e-05,
+      "loss": 1.978,
+      "step": 384
+    },
+    {
+      "epoch": 0.7324478178368121,
+      "grad_norm": 1.841895341873169,
+      "learning_rate": 7.509054458112202e-05,
+      "loss": 2.2376,
+      "step": 386
+    },
+    {
+      "epoch": 0.7362428842504743,
+      "grad_norm": 1.7870166301727295,
+      "learning_rate": 7.481858290490917e-05,
+      "loss": 1.9684,
+      "step": 388
+    },
+    {
+      "epoch": 0.7400379506641366,
+      "grad_norm": 1.8254350423812866,
+      "learning_rate": 7.45456433901879e-05,
+      "loss": 2.0589,
+      "step": 390
+    },
+    {
+      "epoch": 0.7438330170777988,
+      "grad_norm": 1.7817878723144531,
+      "learning_rate": 7.427173679062484e-05,
+      "loss": 2.0451,
+      "step": 392
+    },
+    {
+      "epoch": 0.7476280834914611,
+      "grad_norm": 1.9436228275299072,
+      "learning_rate": 7.399687389798933e-05,
+      "loss": 2.1209,
+      "step": 394
+    },
+    {
+      "epoch": 0.7514231499051234,
+      "grad_norm": 1.7727830410003662,
+      "learning_rate": 7.372106554172802e-05,
+      "loss": 1.997,
+      "step": 396
+    },
+    {
+      "epoch": 0.7552182163187856,
+      "grad_norm": 1.7208454608917236,
+      "learning_rate": 7.344432258853841e-05,
+      "loss": 2.0055,
+      "step": 398
+    },
+    {
+      "epoch": 0.7590132827324478,
+      "grad_norm": 1.7357947826385498,
+      "learning_rate": 7.316665594194053e-05,
+      "loss": 2.0916,
+      "step": 400
+    },
+    {
+      "epoch": 0.7628083491461101,
+      "grad_norm": 1.5990426540374756,
+      "learning_rate": 7.288807654184747e-05,
+      "loss": 2.0385,
+      "step": 402
+    },
+    {
+      "epoch": 0.7666034155597723,
+      "grad_norm": 1.6348865032196045,
+      "learning_rate": 7.260859536413429e-05,
+      "loss": 2.0467,
+      "step": 404
+    },
+    {
+      "epoch": 0.7703984819734345,
+      "grad_norm": 1.546189785003662,
+      "learning_rate": 7.232822342020557e-05,
+      "loss": 1.8996,
+      "step": 406
+    },
+    {
+      "epoch": 0.7741935483870968,
+      "grad_norm": 1.7460343837738037,
+      "learning_rate": 7.204697175656165e-05,
+      "loss": 2.1052,
+      "step": 408
+    },
+    {
+      "epoch": 0.777988614800759,
+      "grad_norm": 1.6906976699829102,
+      "learning_rate": 7.176485145436325e-05,
+      "loss": 2.0588,
+      "step": 410
+    },
+    {
+      "epoch": 0.7817836812144212,
+      "grad_norm": 1.9893949031829834,
+      "learning_rate": 7.148187362899505e-05,
+      "loss": 2.0776,
+      "step": 412
+    },
+    {
+      "epoch": 0.7855787476280834,
+      "grad_norm": 1.6658051013946533,
+      "learning_rate": 7.119804942962762e-05,
+      "loss": 1.8487,
+      "step": 414
+    },
+    {
+      "epoch": 0.7893738140417458,
+      "grad_norm": 1.7261987924575806,
+      "learning_rate": 7.091339003877826e-05,
+      "loss": 2.0895,
+      "step": 416
+    },
+    {
+      "epoch": 0.793168880455408,
+      "grad_norm": 1.6543688774108887,
+      "learning_rate": 7.062790667187029e-05,
+      "loss": 1.9453,
+      "step": 418
+    },
+    {
+      "epoch": 0.7969639468690702,
+      "grad_norm": 1.6983455419540405,
+      "learning_rate": 7.034161057679127e-05,
+      "loss": 1.9985,
+      "step": 420
+    },
+    {
+      "epoch": 0.8007590132827325,
+      "grad_norm": 1.6611037254333496,
+      "learning_rate": 7.005451303344979e-05,
+      "loss": 1.9633,
+      "step": 422
+    },
+    {
+      "epoch": 0.8045540796963947,
+      "grad_norm": 1.979058861732483,
+      "learning_rate": 6.976662535333107e-05,
+      "loss": 1.8833,
+      "step": 424
+    },
+    {
+      "epoch": 0.8083491461100569,
+      "grad_norm": 2.006376266479492,
+      "learning_rate": 6.947795887905127e-05,
+      "loss": 1.8475,
+      "step": 426
+    },
+    {
+      "epoch": 0.8121442125237192,
+      "grad_norm": 1.8918062448501587,
+      "learning_rate": 6.918852498391063e-05,
+      "loss": 1.9098,
+      "step": 428
+    },
+    {
+      "epoch": 0.8159392789373814,
+      "grad_norm": 1.933212161064148,
+      "learning_rate": 6.889833507144532e-05,
+      "loss": 2.1358,
+      "step": 430
+    },
+    {
+      "epoch": 0.8197343453510436,
+      "grad_norm": 1.8028526306152344,
+      "learning_rate": 6.860740057497823e-05,
+      "loss": 1.9799,
+      "step": 432
+    },
+    {
+      "epoch": 0.8235294117647058,
+      "grad_norm": 2.174152374267578,
+      "learning_rate": 6.831573295716837e-05,
+      "loss": 2.0668,
+      "step": 434
+    },
+    {
+      "epoch": 0.8273244781783681,
+      "grad_norm": 2.581892728805542,
+      "learning_rate": 6.802334370955941e-05,
+      "loss": 1.9118,
+      "step": 436
+    },
+    {
+      "epoch": 0.8311195445920304,
+      "grad_norm": 1.7758573293685913,
+      "learning_rate": 6.773024435212678e-05,
+      "loss": 2.0122,
+      "step": 438
+    },
+    {
+      "epoch": 0.8349146110056926,
+      "grad_norm": 1.7795305252075195,
+      "learning_rate": 6.743644643282388e-05,
+      "loss": 2.1017,
+      "step": 440
+    },
+    {
+      "epoch": 0.8387096774193549,
+      "grad_norm": 1.9165265560150146,
+      "learning_rate": 6.714196152712704e-05,
+      "loss": 2.1366,
+      "step": 442
+    },
+    {
+      "epoch": 0.8425047438330171,
+      "grad_norm": 3.430307149887085,
+      "learning_rate": 6.684680123757949e-05,
+      "loss": 1.9484,
+      "step": 444
+    },
+    {
+      "epoch": 0.8462998102466793,
+      "grad_norm": 1.6847901344299316,
+      "learning_rate": 6.65509771933342e-05,
+      "loss": 1.8189,
+      "step": 446
+    },
+    {
+      "epoch": 0.8500948766603416,
+      "grad_norm": 1.6464290618896484,
+      "learning_rate": 6.625450104969572e-05,
+      "loss": 1.9903,
+      "step": 448
+    },
+    {
+      "epoch": 0.8538899430740038,
+      "grad_norm": 1.7267745733261108,
+      "learning_rate": 6.595738448766095e-05,
+      "loss": 1.9767,
+      "step": 450
+    },
+    {
+      "epoch": 0.857685009487666,
+      "grad_norm": 1.9262585639953613,
+      "learning_rate": 6.565963921345895e-05,
+      "loss": 1.9669,
+      "step": 452
+    },
+    {
+      "epoch": 0.8614800759013282,
+      "grad_norm": 1.6643258333206177,
+      "learning_rate": 6.536127695808964e-05,
+      "loss": 1.9634,
+      "step": 454
+    },
+    {
+      "epoch": 0.8652751423149905,
+      "grad_norm": 1.577966332435608,
+      "learning_rate": 6.506230947686172e-05,
+      "loss": 1.8958,
+      "step": 456
+    },
+    {
+      "epoch": 0.8690702087286527,
+      "grad_norm": 2.840637683868408,
+      "learning_rate": 6.47627485489294e-05,
+      "loss": 2.0766,
+      "step": 458
+    },
+    {
+      "epoch": 0.872865275142315,
+      "grad_norm": 1.7307078838348389,
+      "learning_rate": 6.446260597682839e-05,
+      "loss": 1.846,
+      "step": 460
+    },
+    {
+      "epoch": 0.8766603415559773,
+      "grad_norm": 1.7440413236618042,
+      "learning_rate": 6.416189358601088e-05,
+      "loss": 1.8265,
+      "step": 462
+    },
+    {
+      "epoch": 0.8804554079696395,
+      "grad_norm": 1.5109995603561401,
+      "learning_rate": 6.386062322437954e-05,
+      "loss": 1.8226,
+      "step": 464
+    },
+    {
+      "epoch": 0.8842504743833017,
+      "grad_norm": 1.671194076538086,
+      "learning_rate": 6.355880676182086e-05,
+      "loss": 2.038,
+      "step": 466
+    },
+    {
+      "epoch": 0.888045540796964,
+      "grad_norm": 1.5358309745788574,
+      "learning_rate": 6.325645608973735e-05,
+      "loss": 1.9325,
+      "step": 468
+    },
+    {
+      "epoch": 0.8918406072106262,
+      "grad_norm": 1.6306613683700562,
+      "learning_rate": 6.295358312057914e-05,
+      "loss": 1.9589,
+      "step": 470
+    },
+    {
+      "epoch": 0.8956356736242884,
+      "grad_norm": 1.5553851127624512,
+      "learning_rate": 6.26501997873745e-05,
+      "loss": 1.8156,
+      "step": 472
+    },
+    {
+      "epoch": 0.8994307400379506,
+      "grad_norm": 1.6885318756103516,
+      "learning_rate": 6.234631804325981e-05,
+      "loss": 1.9792,
+      "step": 474
+    },
+    {
+      "epoch": 0.9032258064516129,
+      "grad_norm": 5.679869651794434,
+      "learning_rate": 6.204194986100857e-05,
+      "loss": 1.9165,
+      "step": 476
+    },
+    {
+      "epoch": 0.9070208728652751,
+      "grad_norm": 1.6526105403900146,
+      "learning_rate": 6.173710723255966e-05,
+      "loss": 2.0554,
+      "step": 478
+    },
+    {
+      "epoch": 0.9108159392789373,
+      "grad_norm": 1.577702283859253,
+      "learning_rate": 6.143180216854487e-05,
+      "loss": 1.7735,
+      "step": 480
+    },
+    {
+      "epoch": 0.9146110056925996,
+      "grad_norm": 2.24104380607605,
+      "learning_rate": 6.112604669781572e-05,
+      "loss": 1.9426,
+      "step": 482
+    },
+    {
+      "epoch": 0.9184060721062619,
+      "grad_norm": 1.5590384006500244,
+      "learning_rate": 6.081985286696949e-05,
+      "loss": 2.0698,
+      "step": 484
+    },
+    {
+      "epoch": 0.9222011385199241,
+      "grad_norm": 1.7752505540847778,
+      "learning_rate": 6.051323273987463e-05,
+      "loss": 1.889,
+      "step": 486
+    },
+    {
+      "epoch": 0.9259962049335864,
+      "grad_norm": 1.7600066661834717,
+      "learning_rate": 6.020619839719538e-05,
+      "loss": 1.9693,
+      "step": 488
+    },
+    {
+      "epoch": 0.9297912713472486,
+      "grad_norm": 1.5522412061691284,
+      "learning_rate": 5.989876193591589e-05,
+      "loss": 1.9813,
+      "step": 490
+    },
+    {
+      "epoch": 0.9335863377609108,
+      "grad_norm": 2.223928213119507,
+      "learning_rate": 5.959093546886356e-05,
+      "loss": 1.954,
+      "step": 492
+    },
+    {
+      "epoch": 0.937381404174573,
+      "grad_norm": 2.466945171356201,
+      "learning_rate": 5.928273112423177e-05,
+      "loss": 1.7863,
+      "step": 494
+    },
+    {
+      "epoch": 0.9411764705882353,
+      "grad_norm": 1.743293285369873,
+      "learning_rate": 5.897416104510211e-05,
+      "loss": 1.9432,
+      "step": 496
+    },
+    {
+      "epoch": 0.9449715370018975,
+      "grad_norm": 1.6011027097702026,
+      "learning_rate": 5.866523738896587e-05,
+      "loss": 1.8718,
+      "step": 498
+    },
+    {
+      "epoch": 0.9487666034155597,
+      "grad_norm": 1.6478995084762573,
+      "learning_rate": 5.835597232724511e-05,
+      "loss": 1.9411,
+      "step": 500
+    },
+    {
+      "epoch": 0.952561669829222,
+      "grad_norm": 1.6733437776565552,
+      "learning_rate": 5.804637804481306e-05,
+      "loss": 1.8868,
+      "step": 502
+    },
+    {
+      "epoch": 0.9563567362428842,
+      "grad_norm": 1.6197586059570312,
+      "learning_rate": 5.773646673951406e-05,
+      "loss": 1.9805,
+      "step": 504
+    },
+    {
+      "epoch": 0.9601518026565465,
+      "grad_norm": 1.7262201309204102,
+      "learning_rate": 5.742625062168303e-05,
+      "loss": 2.0499,
+      "step": 506
+    },
+    {
+      "epoch": 0.9639468690702088,
+      "grad_norm": 1.5373488664627075,
+      "learning_rate": 5.7115741913664264e-05,
+      "loss": 1.8859,
+      "step": 508
+    },
+    {
+      "epoch": 0.967741935483871,
+      "grad_norm": 2.0669445991516113,
+      "learning_rate": 5.680495284933e-05,
+      "loss": 1.9428,
+      "step": 510
+    },
+    {
+      "epoch": 0.9715370018975332,
+      "grad_norm": 2.279287099838257,
+      "learning_rate": 5.649389567359831e-05,
+      "loss": 1.9444,
+      "step": 512
+    },
+    {
+      "epoch": 0.9753320683111955,
+      "grad_norm": 1.7030115127563477,
+      "learning_rate": 5.6182582641950764e-05,
+      "loss": 1.7837,
+      "step": 514
+    },
+    {
+      "epoch": 0.9791271347248577,
+      "grad_norm": 1.6407768726348877,
+      "learning_rate": 5.58710260199495e-05,
+      "loss": 1.8909,
+      "step": 516
+    },
+    {
+      "epoch": 0.9829222011385199,
+      "grad_norm": 1.5273981094360352,
+      "learning_rate": 5.555923808275395e-05,
+      "loss": 1.736,
+      "step": 518
+    },
+    {
+      "epoch": 0.9867172675521821,
+      "grad_norm": 1.762691617012024,
+      "learning_rate": 5.5247231114637256e-05,
+      "loss": 1.7998,
+      "step": 520
+    },
+    {
+      "epoch": 0.9905123339658444,
+      "grad_norm": 1.6185078620910645,
+      "learning_rate": 5.4935017408502274e-05,
+      "loss": 1.9654,
+      "step": 522
+    },
+    {
+      "epoch": 0.9943074003795066,
+      "grad_norm": 1.666943907737732,
+      "learning_rate": 5.462260926539722e-05,
+      "loss": 1.9379,
+      "step": 524
+    },
+    {
+      "epoch": 0.9981024667931688,
+      "grad_norm": 1.5249921083450317,
+      "learning_rate": 5.431001899403098e-05,
+      "loss": 1.9387,
+      "step": 526
+    },
+    {
+      "epoch": 1.0018975332068312,
+      "grad_norm": 1.6618927717208862,
+      "learning_rate": 5.399725891028824e-05,
+      "loss": 1.6652,
+      "step": 528
+    },
+    {
+      "epoch": 1.0056925996204933,
+      "grad_norm": 1.5493872165679932,
+      "learning_rate": 5.368434133674417e-05,
+      "loss": 1.6482,
+      "step": 530
+    },
+    {
+      "epoch": 1.0094876660341556,
+      "grad_norm": 1.5997267961502075,
+      "learning_rate": 5.3371278602179e-05,
+      "loss": 1.7815,
+      "step": 532
+    },
+    {
+      "epoch": 1.0132827324478177,
+      "grad_norm": 1.6044597625732422,
+      "learning_rate": 5.305808304109214e-05,
+      "loss": 1.7196,
+      "step": 534
+    },
+    {
+      "epoch": 1.01707779886148,
+      "grad_norm": 1.6980633735656738,
+      "learning_rate": 5.274476699321638e-05,
+      "loss": 1.6706,
+      "step": 536
+    },
+    {
+      "epoch": 1.0208728652751424,
+      "grad_norm": 1.4464764595031738,
+      "learning_rate": 5.243134280303156e-05,
+      "loss": 1.6116,
+      "step": 538
+    },
+    {
+      "epoch": 1.0246679316888045,
+      "grad_norm": 1.7325392961502075,
+      "learning_rate": 5.211782281927832e-05,
+      "loss": 1.8553,
+      "step": 540
+    },
+    {
+      "epoch": 1.0284629981024669,
+      "grad_norm": 2.1889193058013916,
+      "learning_rate": 5.1804219394471456e-05,
+      "loss": 1.7341,
+      "step": 542
+    },
+    {
+      "epoch": 1.032258064516129,
+      "grad_norm": 1.5160493850708008,
+      "learning_rate": 5.149054488441333e-05,
+      "loss": 1.8746,
+      "step": 544
+    },
+    {
+      "epoch": 1.0360531309297913,
+      "grad_norm": 1.5881016254425049,
+      "learning_rate": 5.117681164770704e-05,
+      "loss": 1.5998,
+      "step": 546
+    },
+    {
+      "epoch": 1.0398481973434535,
+      "grad_norm": 1.9865212440490723,
+      "learning_rate": 5.086303204526943e-05,
+      "loss": 1.6456,
+      "step": 548
+    },
+    {
+      "epoch": 1.0436432637571158,
+      "grad_norm": 1.6527037620544434,
+      "learning_rate": 5.054921843984418e-05,
+      "loss": 1.5798,
+      "step": 550
+    },
+    {
+      "epoch": 1.047438330170778,
+      "grad_norm": 2.3243703842163086,
+      "learning_rate": 5.023538319551465e-05,
+      "loss": 1.6549,
+      "step": 552
+    },
+    {
+      "epoch": 1.0512333965844403,
+      "grad_norm": 1.6025899648666382,
+      "learning_rate": 4.9921538677216736e-05,
+      "loss": 1.7226,
+      "step": 554
+    },
+    {
+      "epoch": 1.0550284629981024,
+      "grad_norm": 1.9399139881134033,
+      "learning_rate": 4.9607697250251786e-05,
+      "loss": 1.679,
+      "step": 556
+    },
+    {
+      "epoch": 1.0588235294117647,
+      "grad_norm": 1.4900729656219482,
+      "learning_rate": 4.929387127979927e-05,
+      "loss": 1.7285,
+      "step": 558
+    },
+    {
+      "epoch": 1.0626185958254268,
+      "grad_norm": 1.494438648223877,
+      "learning_rate": 4.898007313042975e-05,
+      "loss": 1.7318,
+      "step": 560
+    },
+    {
+      "epoch": 1.0664136622390892,
+      "grad_norm": 1.6275337934494019,
+      "learning_rate": 4.866631516561759e-05,
+      "loss": 1.622,
+      "step": 562
+    },
+    {
+      "epoch": 1.0702087286527515,
+      "grad_norm": 2.017521381378174,
+      "learning_rate": 4.835260974725397e-05,
+      "loss": 1.778,
+      "step": 564
+    },
+    {
+      "epoch": 1.0740037950664136,
+      "grad_norm": 1.5728139877319336,
+      "learning_rate": 4.803896923515974e-05,
+      "loss": 1.6612,
+      "step": 566
+    },
+    {
+      "epoch": 1.077798861480076,
+      "grad_norm": 2.189615488052368,
+      "learning_rate": 4.772540598659844e-05,
+      "loss": 1.8048,
+      "step": 568
+    },
+    {
+      "epoch": 1.081593927893738,
+      "grad_norm": 2.1633052825927734,
+      "learning_rate": 4.741193235578952e-05,
+      "loss": 1.6475,
+      "step": 570
+    },
+    {
+      "epoch": 1.0853889943074004,
+      "grad_norm": 1.6020445823669434,
+      "learning_rate": 4.7098560693421565e-05,
+      "loss": 1.6862,
+      "step": 572
+    },
+    {
+      "epoch": 1.0891840607210626,
+      "grad_norm": 1.6626943349838257,
+      "learning_rate": 4.678530334616557e-05,
+      "loss": 1.7286,
+      "step": 574
+    },
+    {
+      "epoch": 1.092979127134725,
+      "grad_norm": 1.7282367944717407,
+      "learning_rate": 4.647217265618866e-05,
+      "loss": 1.6667,
+      "step": 576
+    },
+    {
+      "epoch": 1.096774193548387,
+      "grad_norm": 1.960288405418396,
+      "learning_rate": 4.615918096066766e-05,
+      "loss": 1.6368,
+      "step": 578
+    },
+    {
+      "epoch": 1.1005692599620494,
+      "grad_norm": 1.614453911781311,
+      "learning_rate": 4.584634059130317e-05,
+      "loss": 1.6,
+      "step": 580
+    },
+    {
+      "epoch": 1.1043643263757117,
+      "grad_norm": 1.5496991872787476,
+      "learning_rate": 4.553366387383357e-05,
+      "loss": 1.6395,
+      "step": 582
+    },
+    {
+      "epoch": 1.1081593927893738,
+      "grad_norm": 1.6079200506210327,
+      "learning_rate": 4.522116312754944e-05,
+      "loss": 1.7718,
+      "step": 584
+    },
+    {
+      "epoch": 1.1119544592030361,
+      "grad_norm": 3.4496216773986816,
+      "learning_rate": 4.4908850664808245e-05,
+      "loss": 1.6899,
+      "step": 586
+    },
+    {
+      "epoch": 1.1157495256166983,
+      "grad_norm": 1.5524213314056396,
+      "learning_rate": 4.4596738790549114e-05,
+      "loss": 1.7743,
+      "step": 588
+    },
+    {
+      "epoch": 1.1195445920303606,
+      "grad_norm": 1.9812953472137451,
+      "learning_rate": 4.428483980180814e-05,
+      "loss": 1.6655,
+      "step": 590
+    },
+    {
+      "epoch": 1.1233396584440227,
+      "grad_norm": 1.5174269676208496,
+      "learning_rate": 4.397316598723385e-05,
+      "loss": 1.6685,
+      "step": 592
+    },
+    {
+      "epoch": 1.127134724857685,
+      "grad_norm": 1.4925235509872437,
+      "learning_rate": 4.366172962660301e-05,
+      "loss": 1.6995,
+      "step": 594
+    },
+    {
+      "epoch": 1.1309297912713472,
+      "grad_norm": 1.4607934951782227,
+      "learning_rate": 4.335054299033686e-05,
+      "loss": 1.7295,
+      "step": 596
+    },
+    {
+      "epoch": 1.1347248576850095,
+      "grad_norm": 1.7591880559921265,
+      "learning_rate": 4.3039618339017595e-05,
+      "loss": 1.5847,
+      "step": 598
+    },
+    {
+      "epoch": 1.1385199240986716,
+      "grad_norm": 1.8234410285949707,
+      "learning_rate": 4.272896792290537e-05,
+      "loss": 1.7632,
+      "step": 600
+    },
+    {
+      "epoch": 1.142314990512334,
+      "grad_norm": 1.6295688152313232,
+      "learning_rate": 4.241860398145565e-05,
+      "loss": 1.6659,
+      "step": 602
+    },
+    {
+      "epoch": 1.146110056925996,
+      "grad_norm": 1.6801655292510986,
+      "learning_rate": 4.2108538742836906e-05,
+      "loss": 1.7868,
+      "step": 604
+    },
+    {
+      "epoch": 1.1499051233396584,
+      "grad_norm": 1.5158848762512207,
+      "learning_rate": 4.179878442344892e-05,
+      "loss": 1.7731,
+      "step": 606
+    },
+    {
+      "epoch": 1.1537001897533208,
+      "grad_norm": 1.7007004022598267,
+      "learning_rate": 4.14893532274414e-05,
+      "loss": 1.5958,
+      "step": 608
+    },
+    {
+      "epoch": 1.157495256166983,
+      "grad_norm": 1.4145736694335938,
+      "learning_rate": 4.1180257346233186e-05,
+      "loss": 1.4984,
+      "step": 610
+    },
+    {
+      "epoch": 1.1612903225806452,
+      "grad_norm": 1.6077505350112915,
+      "learning_rate": 4.0871508958031924e-05,
+      "loss": 1.7519,
+      "step": 612
+    },
+    {
+      "epoch": 1.1650853889943074,
+      "grad_norm": 1.9386768341064453,
+      "learning_rate": 4.056312022735417e-05,
+      "loss": 1.6402,
+      "step": 614
+    },
+    {
+      "epoch": 1.1688804554079697,
+      "grad_norm": 1.5569063425064087,
+      "learning_rate": 4.025510330454621e-05,
+      "loss": 1.667,
+      "step": 616
+    },
+    {
+      "epoch": 1.1726755218216318,
+      "grad_norm": 1.823422908782959,
+      "learning_rate": 3.994747032530532e-05,
+      "loss": 1.8977,
+      "step": 618
+    },
+    {
+      "epoch": 1.1764705882352942,
+      "grad_norm": 1.5259554386138916,
+      "learning_rate": 3.9640233410201553e-05,
+      "loss": 1.5713,
+      "step": 620
+    },
+    {
+      "epoch": 1.1802656546489563,
+      "grad_norm": 1.65779447555542,
+      "learning_rate": 3.933340466420032e-05,
+      "loss": 1.6914,
+      "step": 622
+    },
+    {
+      "epoch": 1.1840607210626186,
+      "grad_norm": 3.1816537380218506,
+      "learning_rate": 3.902699617618531e-05,
+      "loss": 1.6629,
+      "step": 624
+    },
+    {
+      "epoch": 1.187855787476281,
+      "grad_norm": 1.5835717916488647,
+      "learning_rate": 3.872102001848238e-05,
+      "loss": 1.5945,
+      "step": 626
+    },
+    {
+      "epoch": 1.191650853889943,
+      "grad_norm": 1.6983957290649414,
+      "learning_rate": 3.841548824638376e-05,
+      "loss": 1.7144,
+      "step": 628
+    },
+    {
+      "epoch": 1.1954459203036052,
+      "grad_norm": 1.50918710231781,
+      "learning_rate": 3.8110412897673096e-05,
+      "loss": 1.6502,
+      "step": 630
+    },
+    {
+      "epoch": 1.1992409867172675,
+      "grad_norm": 2.0277769565582275,
+      "learning_rate": 3.7805805992151284e-05,
+      "loss": 1.6093,
+      "step": 632
+    },
+    {
+      "epoch": 1.2030360531309299,
+      "grad_norm": 1.649634599685669,
+      "learning_rate": 3.750167953116272e-05,
+      "loss": 1.7188,
+      "step": 634
+    },
+    {
+      "epoch": 1.206831119544592,
+      "grad_norm": 1.4943114519119263,
+      "learning_rate": 3.719804549712265e-05,
+      "loss": 1.638,
+      "step": 636
+    },
+    {
+      "epoch": 1.2106261859582543,
+      "grad_norm": 1.5886467695236206,
+      "learning_rate": 3.689491585304491e-05,
+      "loss": 1.5257,
+      "step": 638
+    },
+    {
+      "epoch": 1.2144212523719164,
+      "grad_norm": 1.7062335014343262,
+      "learning_rate": 3.6592302542070624e-05,
+      "loss": 1.6409,
+      "step": 640
+    },
+    {
+      "epoch": 1.2182163187855788,
+      "grad_norm": 1.4793087244033813,
+      "learning_rate": 3.629021748699777e-05,
+      "loss": 1.6542,
+      "step": 642
+    },
+    {
+      "epoch": 1.222011385199241,
+      "grad_norm": 1.7847223281860352,
+      "learning_rate": 3.598867258981122e-05,
+      "loss": 1.7352,
+      "step": 644
+    },
+    {
+      "epoch": 1.2258064516129032,
+      "grad_norm": 1.4678460359573364,
+      "learning_rate": 3.5687679731214016e-05,
+      "loss": 1.6659,
+      "step": 646
+    },
+    {
+      "epoch": 1.2296015180265654,
+      "grad_norm": 1.5419920682907104,
+      "learning_rate": 3.538725077015915e-05,
+      "loss": 1.6397,
+      "step": 648
+    },
+    {
+      "epoch": 1.2333965844402277,
+      "grad_norm": 1.7071402072906494,
+      "learning_rate": 3.5087397543382326e-05,
+      "loss": 1.7359,
+      "step": 650
+    },
+    {
+      "epoch": 1.23719165085389,
+      "grad_norm": 1.4306018352508545,
+      "learning_rate": 3.478813186493569e-05,
+      "loss": 1.6519,
+      "step": 652
+    },
+    {
+      "epoch": 1.2409867172675522,
+      "grad_norm": 1.5240694284439087,
+      "learning_rate": 3.448946552572229e-05,
+      "loss": 1.7532,
+      "step": 654
+    },
+    {
+      "epoch": 1.2447817836812145,
+      "grad_norm": 1.3893238306045532,
+      "learning_rate": 3.419141029303149e-05,
+      "loss": 1.5148,
+      "step": 656
+    },
+    {
+      "epoch": 1.2485768500948766,
+      "grad_norm": 1.5156265497207642,
+      "learning_rate": 3.389397791007548e-05,
+      "loss": 1.6424,
+      "step": 658
+    },
+    {
+      "epoch": 1.252371916508539,
+      "grad_norm": 1.48092520236969,
+      "learning_rate": 3.3597180095526434e-05,
+      "loss": 1.6634,
+      "step": 660
+    },
+    {
+      "epoch": 1.256166982922201,
+      "grad_norm": 1.4644947052001953,
+      "learning_rate": 3.3301028543054935e-05,
+      "loss": 1.674,
+      "step": 662
+    },
+    {
+      "epoch": 1.2599620493358634,
+      "grad_norm": 1.6219902038574219,
+      "learning_rate": 3.3005534920869176e-05,
+      "loss": 1.7191,
+      "step": 664
+    },
+    {
+      "epoch": 1.2637571157495255,
+      "grad_norm": 1.620972990989685,
+      "learning_rate": 3.271071087125522e-05,
+      "loss": 1.5935,
+      "step": 666
+    },
+    {
+      "epoch": 1.2675521821631879,
+      "grad_norm": 1.5413631200790405,
+      "learning_rate": 3.2416568010118435e-05,
+      "loss": 1.7661,
+      "step": 668
+    },
+    {
+      "epoch": 1.2713472485768502,
+      "grad_norm": 1.5393487215042114,
+      "learning_rate": 3.212311792652564e-05,
+      "loss": 1.5872,
+      "step": 670
+    },
+    {
+      "epoch": 1.2751423149905123,
+      "grad_norm": 1.70036780834198,
+      "learning_rate": 3.183037218224862e-05,
+      "loss": 1.8269,
+      "step": 672
+    },
+    {
+      "epoch": 1.2789373814041745,
+      "grad_norm": 1.5720345973968506,
+      "learning_rate": 3.153834231130866e-05,
+      "loss": 1.7084,
+      "step": 674
+    },
+    {
+      "epoch": 1.2827324478178368,
+      "grad_norm": 1.9221960306167603,
+      "learning_rate": 3.124703981952191e-05,
+      "loss": 1.591,
+      "step": 676
+    },
+    {
+      "epoch": 1.2865275142314991,
+      "grad_norm": 1.7205685377120972,
+      "learning_rate": 3.0956476184046275e-05,
+      "loss": 1.5033,
+      "step": 678
+    },
+    {
+      "epoch": 1.2903225806451613,
+      "grad_norm": 1.5560023784637451,
+      "learning_rate": 3.066666285292906e-05,
+      "loss": 1.6366,
+      "step": 680
+    },
+    {
+      "epoch": 1.2941176470588236,
+      "grad_norm": 1.5918118953704834,
+      "learning_rate": 3.037761124465604e-05,
+      "loss": 1.7091,
+      "step": 682
+    },
+    {
+      "epoch": 1.2979127134724857,
+      "grad_norm": 1.4265037775039673,
+      "learning_rate": 3.0089332747701527e-05,
+      "loss": 1.5692,
+      "step": 684
+    },
+    {
+      "epoch": 1.301707779886148,
+      "grad_norm": 1.8184232711791992,
+      "learning_rate": 2.9801838720079633e-05,
+      "loss": 1.693,
+      "step": 686
+    },
+    {
+      "epoch": 1.3055028462998102,
+      "grad_norm": 1.5686814785003662,
+      "learning_rate": 2.9515140488896847e-05,
+      "loss": 1.5748,
+      "step": 688
+    },
+    {
+      "epoch": 1.3092979127134725,
+      "grad_norm": 1.8368483781814575,
+      "learning_rate": 2.9229249349905684e-05,
+      "loss": 1.7264,
+      "step": 690
+    },
+    {
+      "epoch": 1.3130929791271346,
+      "grad_norm": 1.7172402143478394,
+      "learning_rate": 2.8944176567059706e-05,
+      "loss": 1.6153,
+      "step": 692
+    },
+    {
+      "epoch": 1.316888045540797,
+      "grad_norm": 1.5232385396957397,
+      "learning_rate": 2.86599333720697e-05,
+      "loss": 1.7696,
+      "step": 694
+    },
+    {
+      "epoch": 1.3206831119544593,
+      "grad_norm": 1.7590447664260864,
+      "learning_rate": 2.837653096396108e-05,
+      "loss": 1.5896,
+      "step": 696
+    },
+    {
+      "epoch": 1.3244781783681214,
+      "grad_norm": 1.6392059326171875,
+      "learning_rate": 2.809398050863279e-05,
+      "loss": 1.6988,
+      "step": 698
+    },
+    {
+      "epoch": 1.3282732447817835,
+      "grad_norm": 1.4638209342956543,
+      "learning_rate": 2.7812293138417312e-05,
+      "loss": 1.5469,
+      "step": 700
+    },
+    {
+      "epoch": 1.3320683111954459,
+      "grad_norm": 1.5461550951004028,
+      "learning_rate": 2.7531479951641924e-05,
+      "loss": 1.6668,
+      "step": 702
+    },
+    {
+      "epoch": 1.3358633776091082,
+      "grad_norm": 1.7565243244171143,
+      "learning_rate": 2.7251552012191762e-05,
+      "loss": 1.7475,
+      "step": 704
+    },
+    {
+      "epoch": 1.3396584440227703,
+      "grad_norm": 1.4081997871398926,
+      "learning_rate": 2.697252034907351e-05,
+      "loss": 1.6241,
+      "step": 706
+    },
+    {
+      "epoch": 1.3434535104364327,
+      "grad_norm": 1.4330826997756958,
+      "learning_rate": 2.6694395955981143e-05,
+      "loss": 1.7001,
+      "step": 708
+    },
+    {
+      "epoch": 1.3472485768500948,
+      "grad_norm": 1.534566879272461,
+      "learning_rate": 2.641718979086277e-05,
+      "loss": 1.6876,
+      "step": 710
+    },
+    {
+      "epoch": 1.3510436432637571,
+      "grad_norm": 1.4671937227249146,
+      "learning_rate": 2.614091277548864e-05,
+      "loss": 1.6305,
+      "step": 712
+    },
+    {
+      "epoch": 1.3548387096774195,
+      "grad_norm": 1.687727689743042,
+      "learning_rate": 2.5865575795021218e-05,
+      "loss": 1.7084,
+      "step": 714
+    },
+    {
+      "epoch": 1.3586337760910816,
+      "grad_norm": 1.4534823894500732,
+      "learning_rate": 2.559118969758595e-05,
+      "loss": 1.5755,
+      "step": 716
+    },
+    {
+      "epoch": 1.3624288425047437,
+      "grad_norm": 1.4465699195861816,
+      "learning_rate": 2.531776529384407e-05,
+      "loss": 1.69,
+      "step": 718
+    },
+    {
+      "epoch": 1.366223908918406,
+      "grad_norm": 1.3949697017669678,
+      "learning_rate": 2.504531335656668e-05,
+      "loss": 1.6683,
+      "step": 720
+    },
+    {
+      "epoch": 1.3700189753320684,
+      "grad_norm": 1.3115884065628052,
+      "learning_rate": 2.4773844620210118e-05,
+      "loss": 1.4945,
+      "step": 722
+    },
+    {
+      "epoch": 1.3738140417457305,
+      "grad_norm": 1.5108110904693604,
+      "learning_rate": 2.450336978049322e-05,
+      "loss": 1.6253,
+      "step": 724
+    },
+    {
+      "epoch": 1.3776091081593929,
+      "grad_norm": 1.3981152772903442,
+      "learning_rate": 2.423389949397582e-05,
+      "loss": 1.5409,
+      "step": 726
+    },
+    {
+      "epoch": 1.381404174573055,
+      "grad_norm": 1.5719287395477295,
+      "learning_rate": 2.3965444377638906e-05,
+      "loss": 1.6682,
+      "step": 728
+    },
+    {
+      "epoch": 1.3851992409867173,
+      "grad_norm": 40.58249282836914,
+      "learning_rate": 2.3698015008466372e-05,
+      "loss": 1.5704,
+      "step": 730
+    },
+    {
+      "epoch": 1.3889943074003794,
+      "grad_norm": 1.6211159229278564,
+      "learning_rate": 2.3431621923028145e-05,
+      "loss": 1.6252,
+      "step": 732
+    },
+    {
+      "epoch": 1.3927893738140418,
+      "grad_norm": 1.5556660890579224,
+      "learning_rate": 2.3166275617065213e-05,
+      "loss": 1.7645,
+      "step": 734
+    },
+    {
+      "epoch": 1.396584440227704,
+      "grad_norm": 1.5189040899276733,
+      "learning_rate": 2.2901986545076e-05,
+      "loss": 1.7522,
+      "step": 736
+    },
+    {
+      "epoch": 1.4003795066413662,
+      "grad_norm": 1.4270603656768799,
+      "learning_rate": 2.26387651199045e-05,
+      "loss": 1.6917,
+      "step": 738
+    },
+    {
+      "epoch": 1.4041745730550286,
+      "grad_norm": 1.4696648120880127,
+      "learning_rate": 2.2376621712330015e-05,
+      "loss": 1.6135,
+      "step": 740
+    },
+    {
+      "epoch": 1.4079696394686907,
+      "grad_norm": 1.3072363138198853,
+      "learning_rate": 2.2115566650658536e-05,
+      "loss": 1.585,
+      "step": 742
+    },
+    {
+      "epoch": 1.4117647058823528,
+      "grad_norm": 1.5034172534942627,
+      "learning_rate": 2.185561022031582e-05,
+      "loss": 1.5775,
+      "step": 744
+    },
+    {
+      "epoch": 1.4155597722960152,
+      "grad_norm": 1.512794852256775,
+      "learning_rate": 2.1596762663442218e-05,
+      "loss": 1.643,
+      "step": 746
+    },
+    {
+      "epoch": 1.4193548387096775,
+      "grad_norm": 3.3542797565460205,
+      "learning_rate": 2.1339034178488964e-05,
+      "loss": 1.5938,
+      "step": 748
+    },
+    {
+      "epoch": 1.4231499051233396,
+      "grad_norm": 1.3820551633834839,
+      "learning_rate": 2.1082434919816557e-05,
+      "loss": 1.5712,
+      "step": 750
+    },
+    {
+      "epoch": 1.426944971537002,
+      "grad_norm": 1.6565138101577759,
+      "learning_rate": 2.0826974997294574e-05,
+      "loss": 1.6549,
+      "step": 752
+    },
+    {
+      "epoch": 1.430740037950664,
+      "grad_norm": 1.4901527166366577,
+      "learning_rate": 2.0572664475903386e-05,
+      "loss": 1.4857,
+      "step": 754
+    },
+    {
+      "epoch": 1.4345351043643264,
+      "grad_norm": 1.5091009140014648,
+      "learning_rate": 2.0319513375337578e-05,
+      "loss": 1.5638,
+      "step": 756
+    },
+    {
+      "epoch": 1.4383301707779887,
+      "grad_norm": 1.6802904605865479,
+      "learning_rate": 2.0067531669611206e-05,
+      "loss": 1.6303,
+      "step": 758
+    },
+    {
+      "epoch": 1.4421252371916509,
+      "grad_norm": 1.5811331272125244,
+      "learning_rate": 1.9816729286664798e-05,
+      "loss": 1.6192,
+      "step": 760
+    },
+    {
+      "epoch": 1.445920303605313,
+      "grad_norm": 3.5746853351593018,
+      "learning_rate": 1.9567116107974232e-05,
+      "loss": 1.6014,
+      "step": 762
+    },
+    {
+      "epoch": 1.4497153700189753,
+      "grad_norm": 1.452040195465088,
+      "learning_rate": 1.931870196816138e-05,
+      "loss": 1.5184,
+      "step": 764
+    },
+    {
+      "epoch": 1.4535104364326377,
+      "grad_norm": 1.3807239532470703,
+      "learning_rate": 1.907149665460664e-05,
+      "loss": 1.5317,
+      "step": 766
+    },
+    {
+      "epoch": 1.4573055028462998,
+      "grad_norm": 1.55472993850708,
+      "learning_rate": 1.8825509907063327e-05,
+      "loss": 1.6677,
+      "step": 768
+    },
+    {
+      "epoch": 1.4611005692599621,
+      "grad_norm": 2.8080527782440186,
+      "learning_rate": 1.8580751417273928e-05,
+      "loss": 1.5966,
+      "step": 770
+    },
+    {
+      "epoch": 1.4648956356736242,
+      "grad_norm": 1.4996614456176758,
+      "learning_rate": 1.833723082858825e-05,
+      "loss": 1.5075,
+      "step": 772
+    },
+    {
+      "epoch": 1.4686907020872866,
+      "grad_norm": 1.5204508304595947,
+      "learning_rate": 1.8094957735583463e-05,
+      "loss": 1.6364,
+      "step": 774
+    },
+    {
+      "epoch": 1.4724857685009487,
+      "grad_norm": 1.4906686544418335,
+      "learning_rate": 1.7853941683686114e-05,
+      "loss": 1.561,
+      "step": 776
+    },
+    {
+      "epoch": 1.476280834914611,
+      "grad_norm": 1.5031158924102783,
+      "learning_rate": 1.761419216879601e-05,
+      "loss": 1.7128,
+      "step": 778
+    },
+    {
+      "epoch": 1.4800759013282732,
+      "grad_norm": 1.646573543548584,
+      "learning_rate": 1.7375718636912103e-05,
+      "loss": 1.4712,
+      "step": 780
+    },
+    {
+      "epoch": 1.4838709677419355,
+      "grad_norm": 1.408292293548584,
+      "learning_rate": 1.7138530483760314e-05,
+      "loss": 1.604,
+      "step": 782
+    },
+    {
+      "epoch": 1.4876660341555978,
+      "grad_norm": 1.514613389968872,
+      "learning_rate": 1.690263705442334e-05,
+      "loss": 1.5608,
+      "step": 784
+    },
+    {
+      "epoch": 1.49146110056926,
+      "grad_norm": 1.524839997291565,
+      "learning_rate": 1.6668047642972494e-05,
+      "loss": 1.6983,
+      "step": 786
+    },
+    {
+      "epoch": 1.495256166982922,
+      "grad_norm": 1.4785043001174927,
+      "learning_rate": 1.6434771492101485e-05,
+      "loss": 1.6198,
+      "step": 788
+    },
+    {
+      "epoch": 1.4990512333965844,
+      "grad_norm": 1.3640413284301758,
+      "learning_rate": 1.6202817792762282e-05,
+      "loss": 1.7243,
+      "step": 790
+    },
+    {
+      "epoch": 1.5028462998102468,
+      "grad_norm": 1.5032504796981812,
+      "learning_rate": 1.5972195683802992e-05,
+      "loss": 1.8518,
+      "step": 792
+    },
+    {
+      "epoch": 1.5066413662239089,
+      "grad_norm": 1.4881658554077148,
+      "learning_rate": 1.5742914251607793e-05,
+      "loss": 1.5322,
+      "step": 794
+    },
+    {
+      "epoch": 1.510436432637571,
+      "grad_norm": 1.4453545808792114,
+      "learning_rate": 1.5514982529738946e-05,
+      "loss": 1.7067,
+      "step": 796
+    },
+    {
+      "epoch": 1.5142314990512333,
+      "grad_norm": 1.7550113201141357,
+      "learning_rate": 1.5288409498580824e-05,
+      "loss": 1.8448,
+      "step": 798
+    },
+    {
+      "epoch": 1.5180265654648957,
+      "grad_norm": 1.4147554636001587,
+      "learning_rate": 1.506320408498621e-05,
+      "loss": 1.6183,
+      "step": 800
+    },
+    {
+      "epoch": 1.521821631878558,
+      "grad_norm": 1.3465934991836548,
+      "learning_rate": 1.4839375161924446e-05,
+      "loss": 1.6194,
+      "step": 802
+    },
+    {
+      "epoch": 1.5256166982922201,
+      "grad_norm": 1.3472963571548462,
+      "learning_rate": 1.4616931548131929e-05,
+      "loss": 1.5212,
+      "step": 804
+    },
+    {
+      "epoch": 1.5294117647058822,
+      "grad_norm": 1.2750847339630127,
+      "learning_rate": 1.4395882007764644e-05,
+      "loss": 1.417,
+      "step": 806
+    },
+    {
+      "epoch": 1.5332068311195446,
+      "grad_norm": 1.328033447265625,
+      "learning_rate": 1.4176235250052788e-05,
+      "loss": 1.6971,
+      "step": 808
+    },
+    {
+      "epoch": 1.537001897533207,
+      "grad_norm": 1.6831159591674805,
+      "learning_rate": 1.3957999928957787e-05,
+      "loss": 1.6343,
+      "step": 810
+    },
+    {
+      "epoch": 1.540796963946869,
+      "grad_norm": 1.480717420578003,
+      "learning_rate": 1.3741184642831189e-05,
+      "loss": 1.6074,
+      "step": 812
+    },
+    {
+      "epoch": 1.5445920303605312,
+      "grad_norm": 1.3570265769958496,
+      "learning_rate": 1.3525797934075912e-05,
+      "loss": 1.5549,
+      "step": 814
+    },
+    {
+      "epoch": 1.5483870967741935,
+      "grad_norm": 1.5152511596679688,
+      "learning_rate": 1.3311848288809813e-05,
+      "loss": 1.6743,
+      "step": 816
+    },
+    {
+      "epoch": 1.5521821631878558,
+      "grad_norm": 1.4288028478622437,
+      "learning_rate": 1.3099344136531105e-05,
+      "loss": 1.6527,
+      "step": 818
+    },
+    {
+      "epoch": 1.5559772296015182,
+      "grad_norm": 1.42059326171875,
+      "learning_rate": 1.2888293849786503e-05,
+      "loss": 1.5204,
+      "step": 820
+    },
+    {
+      "epoch": 1.5597722960151803,
+      "grad_norm": 1.530799150466919,
+      "learning_rate": 1.2678705743841152e-05,
+      "loss": 1.6391,
+      "step": 822
+    },
+    {
+      "epoch": 1.5635673624288424,
+      "grad_norm": 1.3500713109970093,
+      "learning_rate": 1.2470588076351036e-05,
+      "loss": 1.4847,
+      "step": 824
+    },
+    {
+      "epoch": 1.5673624288425048,
+      "grad_norm": 1.414480209350586,
+      "learning_rate": 1.2263949047037786e-05,
+      "loss": 1.5067,
+      "step": 826
+    },
+    {
+      "epoch": 1.571157495256167,
+      "grad_norm": 1.3751875162124634,
+      "learning_rate": 1.2058796797365423e-05,
+      "loss": 1.4994,
+      "step": 828
+    },
+    {
+      "epoch": 1.5749525616698292,
+      "grad_norm": 1.3213021755218506,
+      "learning_rate": 1.1855139410219657e-05,
+      "loss": 1.5208,
+      "step": 830
+    },
+    {
+      "epoch": 1.5787476280834913,
+      "grad_norm": 1.3120757341384888,
+      "learning_rate": 1.1652984909589515e-05,
+      "loss": 1.5639,
+      "step": 832
+    },
+    {
+      "epoch": 1.5825426944971537,
+      "grad_norm": 1.4932972192764282,
+      "learning_rate": 1.145234126025102e-05,
+      "loss": 1.5474,
+      "step": 834
+    },
+    {
+      "epoch": 1.586337760910816,
+      "grad_norm": 1.3566076755523682,
+      "learning_rate": 1.1253216367453578e-05,
+      "loss": 1.527,
+      "step": 836
+    },
+    {
+      "epoch": 1.5901328273244781,
+      "grad_norm": 1.4110137224197388,
+      "learning_rate": 1.1055618076608381e-05,
+      "loss": 1.5419,
+      "step": 838
+    },
+    {
+      "epoch": 1.5939278937381403,
+      "grad_norm": 1.3764982223510742,
+      "learning_rate": 1.085955417297932e-05,
+      "loss": 1.514,
+      "step": 840
+    },
+    {
+      "epoch": 1.5977229601518026,
+      "grad_norm": 1.4026317596435547,
+      "learning_rate": 1.0665032381376338e-05,
+      "loss": 1.5618,
+      "step": 842
+    },
+    {
+      "epoch": 1.601518026565465,
+      "grad_norm": 1.3527369499206543,
+      "learning_rate": 1.047206036585095e-05,
+      "loss": 1.5271,
+      "step": 844
+    },
+    {
+      "epoch": 1.6053130929791273,
+      "grad_norm": 1.5318386554718018,
+      "learning_rate": 1.0280645729394366e-05,
+      "loss": 1.6692,
+      "step": 846
+    },
+    {
+      "epoch": 1.6091081593927894,
+      "grad_norm": 1.4500492811203003,
+      "learning_rate": 1.0090796013637965e-05,
+      "loss": 1.6373,
+      "step": 848
+    },
+    {
+      "epoch": 1.6129032258064515,
+      "grad_norm": 1.5734283924102783,
+      "learning_rate": 9.902518698556018e-06,
+      "loss": 1.4735,
+      "step": 850
+    },
+    {
+      "epoch": 1.6166982922201139,
+      "grad_norm": 1.4084815979003906,
+      "learning_rate": 9.715821202171178e-06,
+      "loss": 1.6179,
+      "step": 852
+    },
+    {
+      "epoch": 1.6204933586337762,
+      "grad_norm": 1.3125470876693726,
+      "learning_rate": 9.530710880262023e-06,
+      "loss": 1.5877,
+      "step": 854
+    },
+    {
+      "epoch": 1.6242884250474383,
+      "grad_norm": 2.5337698459625244,
+      "learning_rate": 9.347195026073369e-06,
+      "loss": 1.5603,
+      "step": 856
+    },
+    {
+      "epoch": 1.6280834914611004,
+      "grad_norm": 1.711033821105957,
+      "learning_rate": 9.16528087002892e-06,
+      "loss": 1.6834,
+      "step": 858
+    },
+    {
+      "epoch": 1.6318785578747628,
+      "grad_norm": 1.3725215196609497,
+      "learning_rate": 8.98497557944627e-06,
+      "loss": 1.5769,
+      "step": 860
+    },
+    {
+      "epoch": 1.635673624288425,
+      "grad_norm": 1.2920559644699097,
+      "learning_rate": 8.806286258254664e-06,
+      "loss": 1.5149,
+      "step": 862
+    },
+    {
+      "epoch": 1.6394686907020875,
+      "grad_norm": 2.3137001991271973,
+      "learning_rate": 8.62921994671501e-06,
+      "loss": 1.7416,
+      "step": 864
+    },
+    {
+      "epoch": 1.6432637571157496,
+      "grad_norm": 1.3830606937408447,
+      "learning_rate": 8.453783621142542e-06,
+      "loss": 1.5635,
+      "step": 866
+    },
+    {
+      "epoch": 1.6470588235294117,
+      "grad_norm": 1.4525353908538818,
+      "learning_rate": 8.279984193631967e-06,
+      "loss": 1.5408,
+      "step": 868
+    },
+    {
+      "epoch": 1.650853889943074,
+      "grad_norm": 1.4264976978302002,
+      "learning_rate": 8.10782851178506e-06,
+      "loss": 1.4529,
+      "step": 870
+    },
+    {
+      "epoch": 1.6546489563567364,
+      "grad_norm": 1.4322513341903687,
+      "learning_rate": 7.937323358440935e-06,
+      "loss": 1.5934,
+      "step": 872
+    },
+    {
+      "epoch": 1.6584440227703985,
+      "grad_norm": 1.5460699796676636,
+      "learning_rate": 7.768475451408847e-06,
+      "loss": 1.6017,
+      "step": 874
+    },
+    {
+      "epoch": 1.6622390891840606,
+      "grad_norm": 1.4285356998443604,
+      "learning_rate": 7.601291443203373e-06,
+      "loss": 1.6968,
+      "step": 876
+    },
+    {
+      "epoch": 1.666034155597723,
+      "grad_norm": 15.581119537353516,
+      "learning_rate": 7.435777920782444e-06,
+      "loss": 1.5468,
+      "step": 878
+    },
+    {
+      "epoch": 1.6698292220113853,
+      "grad_norm": 1.3123668432235718,
+      "learning_rate": 7.271941405287763e-06,
+      "loss": 1.4934,
+      "step": 880
+    },
+    {
+      "epoch": 1.6736242884250474,
+      "grad_norm": 1.2993680238723755,
+      "learning_rate": 7.109788351787866e-06,
+      "loss": 1.6143,
+      "step": 882
+    },
+    {
+      "epoch": 1.6774193548387095,
+      "grad_norm": 1.5729187726974487,
+      "learning_rate": 6.949325149023861e-06,
+      "loss": 1.5881,
+      "step": 884
+    },
+    {
+      "epoch": 1.6812144212523719,
+      "grad_norm": 1.3540576696395874,
+      "learning_rate": 6.790558119157597e-06,
+      "loss": 1.5138,
+      "step": 886
+    },
+    {
+      "epoch": 1.6850094876660342,
+      "grad_norm": 1.92926824092865,
+      "learning_rate": 6.633493517522687e-06,
+      "loss": 1.6079,
+      "step": 888
+    },
+    {
+      "epoch": 1.6888045540796965,
+      "grad_norm": 1.5620063543319702,
+      "learning_rate": 6.478137532378004e-06,
+      "loss": 1.5084,
+      "step": 890
+    },
+    {
+      "epoch": 1.6925996204933587,
+      "grad_norm": 1.7819502353668213,
+      "learning_rate": 6.324496284663867e-06,
+      "loss": 1.5027,
+      "step": 892
+    },
+    {
+      "epoch": 1.6963946869070208,
+      "grad_norm": 1.3874931335449219,
+      "learning_rate": 6.172575827760885e-06,
+      "loss": 1.6247,
+      "step": 894
+    },
+    {
+      "epoch": 1.7001897533206831,
+      "grad_norm": 1.3802452087402344,
+      "learning_rate": 6.022382147251454e-06,
+      "loss": 1.5536,
+      "step": 896
+    },
+    {
+      "epoch": 1.7039848197343455,
+      "grad_norm": 1.2863043546676636,
+      "learning_rate": 5.873921160683943e-06,
+      "loss": 1.5286,
+      "step": 898
+    },
+    {
+      "epoch": 1.7077798861480076,
+      "grad_norm": 1.3299353122711182,
+      "learning_rate": 5.727198717339511e-06,
+      "loss": 1.6258,
+      "step": 900
+    },
+    {
+      "epoch": 1.7115749525616697,
+      "grad_norm": 1.3413077592849731,
+      "learning_rate": 5.582220598001681e-06,
+      "loss": 1.6182,
+      "step": 902
+    },
+    {
+      "epoch": 1.715370018975332,
+      "grad_norm": 1.3768631219863892,
+      "learning_rate": 5.438992514728586e-06,
+      "loss": 1.4732,
+      "step": 904
+    },
+    {
+      "epoch": 1.7191650853889944,
+      "grad_norm": 1.4041154384613037,
+      "learning_rate": 5.297520110627868e-06,
+      "loss": 1.6401,
+      "step": 906
+    },
+    {
+      "epoch": 1.7229601518026565,
+      "grad_norm": 1.3863005638122559,
+      "learning_rate": 5.157808959634408e-06,
+      "loss": 1.5701,
+      "step": 908
+    },
+    {
+      "epoch": 1.7267552182163188,
+      "grad_norm": 1.3895708322525024,
+      "learning_rate": 5.0198645662906666e-06,
+      "loss": 1.7111,
+      "step": 910
+    },
+    {
+      "epoch": 1.730550284629981,
+      "grad_norm": 1.4131901264190674,
+      "learning_rate": 4.88369236552983e-06,
+      "loss": 1.5951,
+      "step": 912
+    },
+    {
+      "epoch": 1.7343453510436433,
+      "grad_norm": 1.398961067199707,
+      "learning_rate": 4.74929772246166e-06,
+      "loss": 1.6147,
+      "step": 914
+    },
+    {
+      "epoch": 1.7381404174573056,
+      "grad_norm": 1.401387095451355,
+      "learning_rate": 4.616685932161152e-06,
+      "loss": 1.6099,
+      "step": 916
+    },
+    {
+      "epoch": 1.7419354838709677,
+      "grad_norm": 1.4285647869110107,
+      "learning_rate": 4.4858622194598525e-06,
+      "loss": 1.5869,
+      "step": 918
+    },
+    {
+      "epoch": 1.7457305502846299,
+      "grad_norm": 1.3526160717010498,
+      "learning_rate": 4.356831738740053e-06,
+      "loss": 1.639,
+      "step": 920
+    },
+    {
+      "epoch": 1.7495256166982922,
+      "grad_norm": 1.4206922054290771,
+      "learning_rate": 4.229599573731685e-06,
+      "loss": 1.5643,
+      "step": 922
+    },
+    {
+      "epoch": 1.7533206831119545,
+      "grad_norm": 1.3534013032913208,
+      "learning_rate": 4.1041707373120356e-06,
+      "loss": 1.5566,
+      "step": 924
+    },
+    {
+      "epoch": 1.7571157495256167,
+      "grad_norm": 1.951189398765564,
+      "learning_rate": 3.980550171308228e-06,
+      "loss": 1.5418,
+      "step": 926
+    },
+    {
+      "epoch": 1.7609108159392788,
+      "grad_norm": 1.3316164016723633,
+      "learning_rate": 3.858742746302535e-06,
+      "loss": 1.5778,
+      "step": 928
+    },
+    {
+      "epoch": 1.7647058823529411,
+      "grad_norm": 1.3784003257751465,
+      "learning_rate": 3.738753261440475e-06,
+      "loss": 1.4794,
+      "step": 930
+    },
+    {
+      "epoch": 1.7685009487666035,
+      "grad_norm": 1.3570363521575928,
+      "learning_rate": 3.6205864442417136e-06,
+      "loss": 1.6291,
+      "step": 932
+    },
+    {
+      "epoch": 1.7722960151802658,
+      "grad_norm": 1.3584340810775757,
+      "learning_rate": 3.5042469504138363e-06,
+      "loss": 1.577,
+      "step": 934
+    },
+    {
+      "epoch": 1.776091081593928,
+      "grad_norm": 1.3361767530441284,
+      "learning_rate": 3.3897393636688368e-06,
+      "loss": 1.5577,
+      "step": 936
+    },
+    {
+      "epoch": 1.77988614800759,
+      "grad_norm": 1.3638248443603516,
+      "learning_rate": 3.277068195542654e-06,
+      "loss": 1.4791,
+      "step": 938
+    },
+    {
+      "epoch": 1.7836812144212524,
+      "grad_norm": 1.3946592807769775,
+      "learning_rate": 3.166237885217299e-06,
+      "loss": 1.7023,
+      "step": 940
+    },
+    {
+      "epoch": 1.7874762808349147,
+      "grad_norm": 1.4293212890625,
+      "learning_rate": 3.0572527993460053e-06,
+      "loss": 1.5444,
+      "step": 942
+    },
+    {
+      "epoch": 1.7912713472485768,
+      "grad_norm": 1.3663233518600464,
+      "learning_rate": 2.950117231881183e-06,
+      "loss": 1.5505,
+      "step": 944
+    },
+    {
+      "epoch": 1.795066413662239,
+      "grad_norm": 1.357879400253296,
+      "learning_rate": 2.8448354039052216e-06,
+      "loss": 1.5379,
+      "step": 946
+    },
+    {
+      "epoch": 1.7988614800759013,
+      "grad_norm": 1.3362464904785156,
+      "learning_rate": 2.741411463464211e-06,
+      "loss": 1.6723,
+      "step": 948
+    },
+    {
+      "epoch": 1.8026565464895636,
+      "grad_norm": 1.3323725461959839,
+      "learning_rate": 2.639849485404505e-06,
+      "loss": 1.475,
+      "step": 950
+    },
+    {
+      "epoch": 1.8064516129032258,
+      "grad_norm": 2.3896121978759766,
+      "learning_rate": 2.5401534712121146e-06,
+      "loss": 1.6162,
+      "step": 952
+    },
+    {
+      "epoch": 1.810246679316888,
+      "grad_norm": 1.4233585596084595,
+      "learning_rate": 2.442327348855161e-06,
+      "loss": 1.4973,
+      "step": 954
+    },
+    {
+      "epoch": 1.8140417457305502,
+      "grad_norm": 2.8995790481567383,
+      "learning_rate": 2.3463749726290286e-06,
+      "loss": 1.5579,
+      "step": 956
+    },
+    {
+      "epoch": 1.8178368121442126,
+      "grad_norm": 1.3937581777572632,
+      "learning_rate": 2.252300123004525e-06,
+      "loss": 1.6372,
+      "step": 958
+    },
+    {
+      "epoch": 1.821631878557875,
+      "grad_norm": 1.2747756242752075,
+      "learning_rate": 2.1601065064789704e-06,
+      "loss": 1.48,
+      "step": 960
+    },
+    {
+      "epoch": 1.825426944971537,
+      "grad_norm": 1.3919050693511963,
+      "learning_rate": 2.069797755430097e-06,
+      "loss": 1.4657,
+      "step": 962
+    },
+    {
+      "epoch": 1.8292220113851991,
+      "grad_norm": 1.4085832834243774,
+      "learning_rate": 1.981377427973019e-06,
+      "loss": 1.647,
+      "step": 964
+    },
+    {
+      "epoch": 1.8330170777988615,
+      "grad_norm": 1.4231733083724976,
+      "learning_rate": 1.8948490078199764e-06,
+      "loss": 1.6445,
+      "step": 966
+    },
+    {
+      "epoch": 1.8368121442125238,
+      "grad_norm": 1.4949570894241333,
+      "learning_rate": 1.8102159041430922e-06,
+      "loss": 1.5626,
+      "step": 968
+    },
+    {
+      "epoch": 1.840607210626186,
+      "grad_norm": 1.3209196329116821,
+      "learning_rate": 1.7274814514400994e-06,
+      "loss": 1.5203,
+      "step": 970
+    },
+    {
+      "epoch": 1.844402277039848,
+      "grad_norm": 1.4556705951690674,
+      "learning_rate": 1.6466489094028759e-06,
+      "loss": 1.5087,
+      "step": 972
+    },
+    {
+      "epoch": 1.8481973434535104,
+      "grad_norm": 1.3396247625350952,
+      "learning_rate": 1.5677214627891179e-06,
+      "loss": 1.6808,
+      "step": 974
+    },
+    {
+      "epoch": 1.8519924098671727,
+      "grad_norm": 1.6221957206726074,
+      "learning_rate": 1.4907022212967803e-06,
+      "loss": 1.594,
+      "step": 976
+    },
+    {
+      "epoch": 1.855787476280835,
+      "grad_norm": 1.2195303440093994,
+      "learning_rate": 1.415594219441585e-06,
+      "loss": 1.4427,
+      "step": 978
+    },
+    {
+      "epoch": 1.8595825426944972,
+      "grad_norm": 1.3192423582077026,
+      "learning_rate": 1.3424004164374837e-06,
+      "loss": 1.54,
+      "step": 980
+    },
+    {
+      "epoch": 1.8633776091081593,
+      "grad_norm": 1.3921012878417969,
+      "learning_rate": 1.2711236960800287e-06,
+      "loss": 1.5021,
+      "step": 982
+    },
+    {
+      "epoch": 1.8671726755218216,
+      "grad_norm": 2.0219826698303223,
+      "learning_rate": 1.2017668666327753e-06,
+      "loss": 1.4205,
+      "step": 984
+    },
+    {
+      "epoch": 1.870967741935484,
+      "grad_norm": 1.2949210405349731,
+      "learning_rate": 1.1343326607166527e-06,
+      "loss": 1.5071,
+      "step": 986
+    },
+    {
+      "epoch": 1.874762808349146,
+      "grad_norm": 1.3908876180648804,
+      "learning_rate": 1.0688237352022345e-06,
+      "loss": 1.6749,
+      "step": 988
+    },
+    {
+      "epoch": 1.8785578747628082,
+      "grad_norm": 1.4135441780090332,
+      "learning_rate": 1.0052426711051666e-06,
+      "loss": 1.4801,
+      "step": 990
+    },
+    {
+      "epoch": 1.8823529411764706,
+      "grad_norm": 1.407795786857605,
+      "learning_rate": 9.435919734843645e-07,
+      "loss": 1.4804,
+      "step": 992
+    },
+    {
+      "epoch": 1.886148007590133,
+      "grad_norm": 1.4486916065216064,
+      "learning_rate": 8.83874071343388e-07,
+      "loss": 1.6187,
+      "step": 994
+    },
+    {
+      "epoch": 1.889943074003795,
+      "grad_norm": 1.2793864011764526,
+      "learning_rate": 8.260913175347273e-07,
+      "loss": 1.4298,
+      "step": 996
+    },
+    {
+      "epoch": 1.8937381404174574,
+      "grad_norm": 1.2875016927719116,
+      "learning_rate": 7.702459886670788e-07,
+      "loss": 1.6438,
+      "step": 998
+    },
+    {
+      "epoch": 1.8975332068311195,
+      "grad_norm": 1.3841239213943481,
+      "learning_rate": 7.163402850156653e-07,
+      "loss": 1.6605,
+      "step": 1000
+    },
+    {
+      "epoch": 1.9013282732447818,
+      "grad_norm": 1.3545821905136108,
+      "learning_rate": 6.643763304355566e-07,
+      "loss": 1.5845,
+      "step": 1002
+    },
+    {
+      "epoch": 1.9051233396584442,
+      "grad_norm": 1.5016220808029175,
+      "learning_rate": 6.143561722779523e-07,
+      "loss": 1.6205,
+      "step": 1004
+    },
+    {
+      "epoch": 1.9089184060721063,
+      "grad_norm": 1.4386885166168213,
+      "learning_rate": 5.662817813095633e-07,
+      "loss": 1.6289,
+      "step": 1006
+    },
+    {
+      "epoch": 1.9127134724857684,
+      "grad_norm": 1.3440179824829102,
+      "learning_rate": 5.201550516349407e-07,
+      "loss": 1.6306,
+      "step": 1008
+    },
+    {
+      "epoch": 1.9165085388994307,
+      "grad_norm": 1.3130792379379272,
+      "learning_rate": 4.7597780062184073e-07,
+      "loss": 1.5329,
+      "step": 1010
+    },
+    {
+      "epoch": 1.920303605313093,
+      "grad_norm": 1.3677467107772827,
+      "learning_rate": 4.337517688296544e-07,
+      "loss": 1.5104,
+      "step": 1012
+    },
+    {
+      "epoch": 1.9240986717267552,
+      "grad_norm": 1.3812295198440552,
+      "learning_rate": 3.9347861994078474e-07,
+      "loss": 1.6983,
+      "step": 1014
+    },
+    {
+      "epoch": 1.9278937381404173,
+      "grad_norm": 1.415624737739563,
+      "learning_rate": 3.551599406951434e-07,
+      "loss": 1.6901,
+      "step": 1016
+    },
+    {
+      "epoch": 1.9316888045540797,
+      "grad_norm": 1.738711953163147,
+      "learning_rate": 3.1879724082760076e-07,
+      "loss": 1.61,
+      "step": 1018
+    },
+    {
+      "epoch": 1.935483870967742,
+      "grad_norm": 1.5385488271713257,
+      "learning_rate": 2.843919530085226e-07,
+      "loss": 1.6271,
+      "step": 1020
+    },
+    {
+      "epoch": 1.9392789373814043,
+      "grad_norm": 1.3073030710220337,
+      "learning_rate": 2.51945432787315e-07,
+      "loss": 1.5319,
+      "step": 1022
+    },
+    {
+      "epoch": 1.9430740037950665,
+      "grad_norm": 1.30621337890625,
+      "learning_rate": 2.21458958539017e-07,
+      "loss": 1.5976,
+      "step": 1024
+    },
+    {
+      "epoch": 1.9468690702087286,
+      "grad_norm": 1.7373992204666138,
+      "learning_rate": 1.9293373141394122e-07,
+      "loss": 1.6985,
+      "step": 1026
+    },
+    {
+      "epoch": 1.950664136622391,
+      "grad_norm": 1.3110527992248535,
+      "learning_rate": 1.6637087529033923e-07,
+      "loss": 1.6146,
+      "step": 1028
+    },
+    {
+      "epoch": 1.9544592030360532,
+      "grad_norm": 1.3716634511947632,
+      "learning_rate": 1.4177143673011484e-07,
+      "loss": 1.6099,
+      "step": 1030
+    },
+    {
+      "epoch": 1.9582542694497154,
+      "grad_norm": 1.3418036699295044,
+      "learning_rate": 1.191363849376237e-07,
+      "loss": 1.6057,
+      "step": 1032
+    },
+    {
+      "epoch": 1.9620493358633775,
+      "grad_norm": 1.314018726348877,
+      "learning_rate": 9.846661172143723e-08,
+      "loss": 1.5356,
+      "step": 1034
+    },
+    {
+      "epoch": 1.9658444022770398,
+      "grad_norm": 1.330483078956604,
+      "learning_rate": 7.976293145924852e-08,
+      "loss": 1.7157,
+      "step": 1036
+    },
+    {
+      "epoch": 1.9696394686907022,
+      "grad_norm": 1.333574652671814,
+      "learning_rate": 6.302608106577568e-08,
+      "loss": 1.549,
+      "step": 1038
+    },
+    {
+      "epoch": 1.9734345351043643,
+      "grad_norm": 1.4075253009796143,
+      "learning_rate": 4.825671996370185e-08,
+      "loss": 1.5625,
+      "step": 1040
+    },
+    {
+      "epoch": 1.9772296015180264,
+      "grad_norm": 1.7305724620819092,
+      "learning_rate": 3.545543005773477e-08,
+      "loss": 1.67,
+      "step": 1042
+    },
+    {
+      "epoch": 1.9810246679316887,
+      "grad_norm": 1.3363792896270752,
+      "learning_rate": 2.4622715711647426e-08,
+      "loss": 1.4922,
+      "step": 1044
+    },
+    {
+      "epoch": 1.984819734345351,
+      "grad_norm": 1.3556599617004395,
+      "learning_rate": 1.5759003728427202e-08,
+      "loss": 1.6319,
+      "step": 1046
+    },
+    {
+      "epoch": 1.9886148007590134,
+      "grad_norm": 1.453833818435669,
+      "learning_rate": 8.864643333450495e-09,
+      "loss": 1.5772,
+      "step": 1048
+    },
+    {
+      "epoch": 1.9924098671726755,
+      "grad_norm": 1.3945348262786865,
+      "learning_rate": 3.9399061607103825e-09,
+      "loss": 1.5458,
+      "step": 1050
+    },
+    {
+      "epoch": 1.9962049335863377,
+      "grad_norm": 1.4707152843475342,
+      "learning_rate": 9.84986242158481e-10,
+      "loss": 1.6735,
+      "step": 1052
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.9164336919784546,
+      "learning_rate": 0.0,
+      "loss": 1.5462,
+      "step": 1054
+    }
+  ],
+  "logging_steps": 2,
+  "max_steps": 1054,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 527,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.7824370872496947e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1054/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20386650688791ad0c29b49ae8cdcf0798f5d0868194e8c0a536314658aafe5a
+size 5432

checkpoint-527/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/OLMo-2-1124-7B-mni_Beng-coli-rand/",
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100384
+}

checkpoint-527/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 100257,
+  "pad_token_id": 100277,
+  "transformers_version": "4.48.0.dev0"
+}

checkpoint-527/model-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf05e46adda2632cb57e1073181e11a642f35accd6b19bc85b48b46b02c488b3
+size 4950070472

checkpoint-527/model-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0b6a357d03676c4986588aaf4dc96afc3f44a0498514961d2b2cfa1940655e
+size 4991622816

checkpoint-527/model-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1038a7d346a681a8fb6949e054c2246f3f3c80e826855c1cec6af1155cc607d
+size 4924546896

checkpoint-527/model-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c0a21146ace02d36928787ae7f34be868f7c81b894dea98ec7b0e5a3ea01b2
+size 4857404928

checkpoint-527/model-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f82a3014ccfc0f67208f63785ae4bc71f09aba86db48bf6139d18178abbfed
+size 4857404928

checkpoint-527/model-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ac37b035f5b3b77dccf772c1635bcd3a14fecffcbe71846266efe79d3b1aed0
+size 4614509080

checkpoint-527/model.safetensors.index.json

@@ -0,0 +1,362 @@
+{
+  "metadata": {
+    "total_size": 29195517952
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

checkpoint-527/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdc1efd17ae54e5b17650dce9d36a03c2ce0b552933aca9331857467c2ab67b5
+size 13055369358

checkpoint-527/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-527/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4988f45d958973fdae7bab9a472bd46eb48f56f526b60df9b90e4e353c03eae1
+size 1064

checkpoint-527/trainer_state.json

@@ -0,0 +1,1874 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 527,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003795066413662239,
+      "grad_norm": 98.7540512084961,
+      "learning_rate": 3.7735849056603773e-06,
+      "loss": 14.2228,
+      "step": 2
+    },
+    {
+      "epoch": 0.007590132827324478,
+      "grad_norm": 99.5528564453125,
+      "learning_rate": 7.547169811320755e-06,
+      "loss": 13.3495,
+      "step": 4
+    },
+    {
+      "epoch": 0.011385199240986717,
+      "grad_norm": 83.69811248779297,
+      "learning_rate": 1.1320754716981132e-05,
+      "loss": 12.5304,
+      "step": 6
+    },
+    {
+      "epoch": 0.015180265654648957,
+      "grad_norm": 47.37470626831055,
+      "learning_rate": 1.509433962264151e-05,
+      "loss": 9.4493,
+      "step": 8
+    },
+    {
+      "epoch": 0.018975332068311195,
+      "grad_norm": 31.21275520324707,
+      "learning_rate": 1.8867924528301888e-05,
+      "loss": 8.0006,
+      "step": 10
+    },
+    {
+      "epoch": 0.022770398481973434,
+      "grad_norm": 15.287549018859863,
+      "learning_rate": 2.2641509433962265e-05,
+      "loss": 6.4803,
+      "step": 12
+    },
+    {
+      "epoch": 0.026565464895635674,
+      "grad_norm": 50.968772888183594,
+      "learning_rate": 2.641509433962264e-05,
+      "loss": 6.7702,
+      "step": 14
+    },
+    {
+      "epoch": 0.030360531309297913,
+      "grad_norm": 28.34760856628418,
+      "learning_rate": 3.018867924528302e-05,
+      "loss": 6.2775,
+      "step": 16
+    },
+    {
+      "epoch": 0.03415559772296015,
+      "grad_norm": 24.7742862701416,
+      "learning_rate": 3.39622641509434e-05,
+      "loss": 5.9268,
+      "step": 18
+    },
+    {
+      "epoch": 0.03795066413662239,
+      "grad_norm": 62.243553161621094,
+      "learning_rate": 3.7735849056603776e-05,
+      "loss": 5.4478,
+      "step": 20
+    },
+    {
+      "epoch": 0.04174573055028463,
+      "grad_norm": 20.099931716918945,
+      "learning_rate": 4.150943396226415e-05,
+      "loss": 5.2172,
+      "step": 22
+    },
+    {
+      "epoch": 0.04554079696394687,
+      "grad_norm": 14.098124504089355,
+      "learning_rate": 4.528301886792453e-05,
+      "loss": 4.9133,
+      "step": 24
+    },
+    {
+      "epoch": 0.04933586337760911,
+      "grad_norm": 7.112690448760986,
+      "learning_rate": 4.9056603773584906e-05,
+      "loss": 4.7982,
+      "step": 26
+    },
+    {
+      "epoch": 0.05313092979127135,
+      "grad_norm": 13.210295677185059,
+      "learning_rate": 5.283018867924528e-05,
+      "loss": 4.7099,
+      "step": 28
+    },
+    {
+      "epoch": 0.056925996204933584,
+      "grad_norm": 68.6553726196289,
+      "learning_rate": 5.660377358490566e-05,
+      "loss": 4.3044,
+      "step": 30
+    },
+    {
+      "epoch": 0.06072106261859583,
+      "grad_norm": 7.807187557220459,
+      "learning_rate": 6.037735849056604e-05,
+      "loss": 4.2902,
+      "step": 32
+    },
+    {
+      "epoch": 0.06451612903225806,
+      "grad_norm": 24.964153289794922,
+      "learning_rate": 6.415094339622641e-05,
+      "loss": 4.0632,
+      "step": 34
+    },
+    {
+      "epoch": 0.0683111954459203,
+      "grad_norm": 5.0253095626831055,
+      "learning_rate": 6.79245283018868e-05,
+      "loss": 4.0919,
+      "step": 36
+    },
+    {
+      "epoch": 0.07210626185958255,
+      "grad_norm": 10.098166465759277,
+      "learning_rate": 7.169811320754717e-05,
+      "loss": 3.9745,
+      "step": 38
+    },
+    {
+      "epoch": 0.07590132827324478,
+      "grad_norm": 6.226728439331055,
+      "learning_rate": 7.547169811320755e-05,
+      "loss": 4.0257,
+      "step": 40
+    },
+    {
+      "epoch": 0.07969639468690702,
+      "grad_norm": 9.49015998840332,
+      "learning_rate": 7.924528301886794e-05,
+      "loss": 3.6716,
+      "step": 42
+    },
+    {
+      "epoch": 0.08349146110056926,
+      "grad_norm": 8.414839744567871,
+      "learning_rate": 8.30188679245283e-05,
+      "loss": 3.8115,
+      "step": 44
+    },
+    {
+      "epoch": 0.0872865275142315,
+      "grad_norm": 7.90946102142334,
+      "learning_rate": 8.679245283018869e-05,
+      "loss": 3.8998,
+      "step": 46
+    },
+    {
+      "epoch": 0.09108159392789374,
+      "grad_norm": 6.811845302581787,
+      "learning_rate": 9.056603773584906e-05,
+      "loss": 3.72,
+      "step": 48
+    },
+    {
+      "epoch": 0.09487666034155598,
+      "grad_norm": 8.63953971862793,
+      "learning_rate": 9.433962264150944e-05,
+      "loss": 3.8366,
+      "step": 50
+    },
+    {
+      "epoch": 0.09867172675521822,
+      "grad_norm": 5.25341272354126,
+      "learning_rate": 9.811320754716981e-05,
+      "loss": 3.6955,
+      "step": 52
+    },
+    {
+      "epoch": 0.10246679316888045,
+      "grad_norm": 6.284265041351318,
+      "learning_rate": 9.999975375283309e-05,
+      "loss": 3.5963,
+      "step": 54
+    },
+    {
+      "epoch": 0.1062618595825427,
+      "grad_norm": 8.026937484741211,
+      "learning_rate": 9.999778379005078e-05,
+      "loss": 3.668,
+      "step": 56
+    },
+    {
+      "epoch": 0.11005692599620494,
+      "grad_norm": 9.422395706176758,
+      "learning_rate": 9.99938439421016e-05,
+      "loss": 3.6292,
+      "step": 58
+    },
+    {
+      "epoch": 0.11385199240986717,
+      "grad_norm": 3.755744695663452,
+      "learning_rate": 9.99879343642134e-05,
+      "loss": 3.5123,
+      "step": 60
+    },
+    {
+      "epoch": 0.11764705882352941,
+      "grad_norm": 8.59239387512207,
+      "learning_rate": 9.99800552892203e-05,
+      "loss": 3.5657,
+      "step": 62
+    },
+    {
+      "epoch": 0.12144212523719165,
+      "grad_norm": 6.495412826538086,
+      "learning_rate": 9.997020702755353e-05,
+      "loss": 3.4943,
+      "step": 64
+    },
+    {
+      "epoch": 0.1252371916508539,
+      "grad_norm": 4.058749675750732,
+      "learning_rate": 9.995838996722914e-05,
+      "loss": 3.3784,
+      "step": 66
+    },
+    {
+      "epoch": 0.12903225806451613,
+      "grad_norm": 5.1243696212768555,
+      "learning_rate": 9.994460457383284e-05,
+      "loss": 3.5299,
+      "step": 68
+    },
+    {
+      "epoch": 0.13282732447817835,
+      "grad_norm": 3.1002564430236816,
+      "learning_rate": 9.992885139050154e-05,
+      "loss": 3.233,
+      "step": 70
+    },
+    {
+      "epoch": 0.1366223908918406,
+      "grad_norm": 9.574880599975586,
+      "learning_rate": 9.991113103790198e-05,
+      "loss": 3.3179,
+      "step": 72
+    },
+    {
+      "epoch": 0.14041745730550284,
+      "grad_norm": 4.202469825744629,
+      "learning_rate": 9.98914442142063e-05,
+      "loss": 3.2057,
+      "step": 74
+    },
+    {
+      "epoch": 0.1442125237191651,
+      "grad_norm": 9.224685668945312,
+      "learning_rate": 9.986979169506453e-05,
+      "loss": 3.2404,
+      "step": 76
+    },
+    {
+      "epoch": 0.14800759013282733,
+      "grad_norm": 6.027818202972412,
+      "learning_rate": 9.9846174333574e-05,
+      "loss": 3.3198,
+      "step": 78
+    },
+    {
+      "epoch": 0.15180265654648956,
+      "grad_norm": 5.427716255187988,
+      "learning_rate": 9.982059306024577e-05,
+      "loss": 3.2348,
+      "step": 80
+    },
+    {
+      "epoch": 0.1555977229601518,
+      "grad_norm": 3.3449645042419434,
+      "learning_rate": 9.979304888296792e-05,
+      "loss": 3.1103,
+      "step": 82
+    },
+    {
+      "epoch": 0.15939278937381404,
+      "grad_norm": 3.7018656730651855,
+      "learning_rate": 9.976354288696588e-05,
+      "loss": 3.0045,
+      "step": 84
+    },
+    {
+      "epoch": 0.16318785578747627,
+      "grad_norm": 3.1451151371002197,
+      "learning_rate": 9.973207623475965e-05,
+      "loss": 3.2071,
+      "step": 86
+    },
+    {
+      "epoch": 0.16698292220113853,
+      "grad_norm": 3.133887529373169,
+      "learning_rate": 9.9698650166118e-05,
+      "loss": 3.0677,
+      "step": 88
+    },
+    {
+      "epoch": 0.17077798861480076,
+      "grad_norm": 5.542338848114014,
+      "learning_rate": 9.966326599800967e-05,
+      "loss": 3.0122,
+      "step": 90
+    },
+    {
+      "epoch": 0.174573055028463,
+      "grad_norm": 3.292200803756714,
+      "learning_rate": 9.962592512455138e-05,
+      "loss": 2.9215,
+      "step": 92
+    },
+    {
+      "epoch": 0.17836812144212524,
+      "grad_norm": 3.4283339977264404,
+      "learning_rate": 9.958662901695303e-05,
+      "loss": 2.9141,
+      "step": 94
+    },
+    {
+      "epoch": 0.18216318785578747,
+      "grad_norm": 3.9418880939483643,
+      "learning_rate": 9.954537922345961e-05,
+      "loss": 2.7547,
+      "step": 96
+    },
+    {
+      "epoch": 0.1859582542694497,
+      "grad_norm": 3.3023762702941895,
+      "learning_rate": 9.950217736929029e-05,
+      "loss": 2.8594,
+      "step": 98
+    },
+    {
+      "epoch": 0.18975332068311196,
+      "grad_norm": 3.1366522312164307,
+      "learning_rate": 9.945702515657434e-05,
+      "loss": 2.8524,
+      "step": 100
+    },
+    {
+      "epoch": 0.1935483870967742,
+      "grad_norm": 2.613105535507202,
+      "learning_rate": 9.940992436428409e-05,
+      "loss": 2.8663,
+      "step": 102
+    },
+    {
+      "epoch": 0.19734345351043645,
+      "grad_norm": 5.562861442565918,
+      "learning_rate": 9.936087684816486e-05,
+      "loss": 3.1047,
+      "step": 104
+    },
+    {
+      "epoch": 0.20113851992409867,
+      "grad_norm": 4.076806545257568,
+      "learning_rate": 9.930988454066177e-05,
+      "loss": 2.7816,
+      "step": 106
+    },
+    {
+      "epoch": 0.2049335863377609,
+      "grad_norm": 2.7189831733703613,
+      "learning_rate": 9.92569494508437e-05,
+      "loss": 2.8001,
+      "step": 108
+    },
+    {
+      "epoch": 0.20872865275142316,
+      "grad_norm": 4.94251012802124,
+      "learning_rate": 9.920207366432402e-05,
+      "loss": 2.7223,
+      "step": 110
+    },
+    {
+      "epoch": 0.2125237191650854,
+      "grad_norm": 2.9818546772003174,
+      "learning_rate": 9.914525934317855e-05,
+      "loss": 2.8046,
+      "step": 112
+    },
+    {
+      "epoch": 0.21631878557874762,
+      "grad_norm": 3.87678599357605,
+      "learning_rate": 9.908650872586029e-05,
+      "loss": 2.7476,
+      "step": 114
+    },
+    {
+      "epoch": 0.22011385199240988,
+      "grad_norm": 2.9942760467529297,
+      "learning_rate": 9.90258241271112e-05,
+      "loss": 2.6728,
+      "step": 116
+    },
+    {
+      "epoch": 0.2239089184060721,
+      "grad_norm": 4.912055969238281,
+      "learning_rate": 9.896320793787106e-05,
+      "loss": 2.6644,
+      "step": 118
+    },
+    {
+      "epoch": 0.22770398481973433,
+      "grad_norm": 3.1557791233062744,
+      "learning_rate": 9.889866262518331e-05,
+      "loss": 2.7656,
+      "step": 120
+    },
+    {
+      "epoch": 0.2314990512333966,
+      "grad_norm": 8.984029769897461,
+      "learning_rate": 9.883219073209772e-05,
+      "loss": 2.7668,
+      "step": 122
+    },
+    {
+      "epoch": 0.23529411764705882,
+      "grad_norm": 6.998969078063965,
+      "learning_rate": 9.876379487757034e-05,
+      "loss": 2.7245,
+      "step": 124
+    },
+    {
+      "epoch": 0.23908918406072105,
+      "grad_norm": 2.730355978012085,
+      "learning_rate": 9.869347775636015e-05,
+      "loss": 2.7342,
+      "step": 126
+    },
+    {
+      "epoch": 0.2428842504743833,
+      "grad_norm": 8.605497360229492,
+      "learning_rate": 9.862124213892304e-05,
+      "loss": 2.776,
+      "step": 128
+    },
+    {
+      "epoch": 0.24667931688804554,
+      "grad_norm": 5.028979301452637,
+      "learning_rate": 9.85470908713026e-05,
+      "loss": 2.722,
+      "step": 130
+    },
+    {
+      "epoch": 0.2504743833017078,
+      "grad_norm": 3.3529725074768066,
+      "learning_rate": 9.847102687501797e-05,
+      "loss": 2.7299,
+      "step": 132
+    },
+    {
+      "epoch": 0.25426944971537,
+      "grad_norm": 4.99077844619751,
+      "learning_rate": 9.839305314694873e-05,
+      "loss": 2.7569,
+      "step": 134
+    },
+    {
+      "epoch": 0.25806451612903225,
+      "grad_norm": 5.339693546295166,
+      "learning_rate": 9.831317275921685e-05,
+      "loss": 2.579,
+      "step": 136
+    },
+    {
+      "epoch": 0.2618595825426945,
+      "grad_norm": 5.73229455947876,
+      "learning_rate": 9.823138885906566e-05,
+      "loss": 2.5465,
+      "step": 138
+    },
+    {
+      "epoch": 0.2656546489563567,
+      "grad_norm": 2.466779947280884,
+      "learning_rate": 9.814770466873585e-05,
+      "loss": 2.7257,
+      "step": 140
+    },
+    {
+      "epoch": 0.269449715370019,
+      "grad_norm": 4.790260314941406,
+      "learning_rate": 9.806212348533838e-05,
+      "loss": 2.545,
+      "step": 142
+    },
+    {
+      "epoch": 0.2732447817836812,
+      "grad_norm": 4.156448841094971,
+      "learning_rate": 9.797464868072488e-05,
+      "loss": 2.6578,
+      "step": 144
+    },
+    {
+      "epoch": 0.27703984819734345,
+      "grad_norm": 2.7191035747528076,
+      "learning_rate": 9.788528370135443e-05,
+      "loss": 2.5779,
+      "step": 146
+    },
+    {
+      "epoch": 0.2808349146110057,
+      "grad_norm": 2.6859488487243652,
+      "learning_rate": 9.77940320681581e-05,
+      "loss": 2.5819,
+      "step": 148
+    },
+    {
+      "epoch": 0.2846299810246679,
+      "grad_norm": 3.9940454959869385,
+      "learning_rate": 9.77008973764e-05,
+      "loss": 2.417,
+      "step": 150
+    },
+    {
+      "epoch": 0.2884250474383302,
+      "grad_norm": 2.97194504737854,
+      "learning_rate": 9.760588329553571e-05,
+      "loss": 2.4018,
+      "step": 152
+    },
+    {
+      "epoch": 0.2922201138519924,
+      "grad_norm": 4.698624134063721,
+      "learning_rate": 9.750899356906775e-05,
+      "loss": 2.5237,
+      "step": 154
+    },
+    {
+      "epoch": 0.29601518026565465,
+      "grad_norm": 3.8055150508880615,
+      "learning_rate": 9.741023201439803e-05,
+      "loss": 2.4408,
+      "step": 156
+    },
+    {
+      "epoch": 0.2998102466793169,
+      "grad_norm": 3.4264442920684814,
+      "learning_rate": 9.730960252267743e-05,
+      "loss": 2.5248,
+      "step": 158
+    },
+    {
+      "epoch": 0.3036053130929791,
+      "grad_norm": 2.7180609703063965,
+      "learning_rate": 9.720710905865256e-05,
+      "loss": 2.4598,
+      "step": 160
+    },
+    {
+      "epoch": 0.30740037950664134,
+      "grad_norm": 4.305523872375488,
+      "learning_rate": 9.710275566050951e-05,
+      "loss": 2.5028,
+      "step": 162
+    },
+    {
+      "epoch": 0.3111954459203036,
+      "grad_norm": 2.756655693054199,
+      "learning_rate": 9.699654643971472e-05,
+      "loss": 2.5315,
+      "step": 164
+    },
+    {
+      "epoch": 0.31499051233396586,
+      "grad_norm": 2.449230432510376,
+      "learning_rate": 9.688848558085306e-05,
+      "loss": 2.3747,
+      "step": 166
+    },
+    {
+      "epoch": 0.3187855787476281,
+      "grad_norm": 3.7404186725616455,
+      "learning_rate": 9.677857734146289e-05,
+      "loss": 2.4413,
+      "step": 168
+    },
+    {
+      "epoch": 0.3225806451612903,
+      "grad_norm": 2.4831082820892334,
+      "learning_rate": 9.666682605186835e-05,
+      "loss": 2.5693,
+      "step": 170
+    },
+    {
+      "epoch": 0.32637571157495254,
+      "grad_norm": 3.9603495597839355,
+      "learning_rate": 9.655323611500875e-05,
+      "loss": 2.603,
+      "step": 172
+    },
+    {
+      "epoch": 0.3301707779886148,
+      "grad_norm": 2.5853846073150635,
+      "learning_rate": 9.643781200626511e-05,
+      "loss": 2.3858,
+      "step": 174
+    },
+    {
+      "epoch": 0.33396584440227706,
+      "grad_norm": 2.5267770290374756,
+      "learning_rate": 9.632055827328382e-05,
+      "loss": 2.4411,
+      "step": 176
+    },
+    {
+      "epoch": 0.3377609108159393,
+      "grad_norm": 2.9036340713500977,
+      "learning_rate": 9.620147953579737e-05,
+      "loss": 2.4533,
+      "step": 178
+    },
+    {
+      "epoch": 0.3415559772296015,
+      "grad_norm": 2.319636344909668,
+      "learning_rate": 9.608058048544251e-05,
+      "loss": 2.4877,
+      "step": 180
+    },
+    {
+      "epoch": 0.34535104364326374,
+      "grad_norm": 3.325531244277954,
+      "learning_rate": 9.595786588557532e-05,
+      "loss": 2.4027,
+      "step": 182
+    },
+    {
+      "epoch": 0.349146110056926,
+      "grad_norm": 11.142248153686523,
+      "learning_rate": 9.583334057108346e-05,
+      "loss": 2.3184,
+      "step": 184
+    },
+    {
+      "epoch": 0.35294117647058826,
+      "grad_norm": 2.423501968383789,
+      "learning_rate": 9.570700944819584e-05,
+      "loss": 2.5982,
+      "step": 186
+    },
+    {
+      "epoch": 0.3567362428842505,
+      "grad_norm": 2.4559223651885986,
+      "learning_rate": 9.557887749428913e-05,
+      "loss": 2.4276,
+      "step": 188
+    },
+    {
+      "epoch": 0.3605313092979127,
+      "grad_norm": 2.703495502471924,
+      "learning_rate": 9.544894975769186e-05,
+      "loss": 2.3763,
+      "step": 190
+    },
+    {
+      "epoch": 0.36432637571157495,
+      "grad_norm": 1.9051834344863892,
+      "learning_rate": 9.531723135748529e-05,
+      "loss": 2.3381,
+      "step": 192
+    },
+    {
+      "epoch": 0.3681214421252372,
+      "grad_norm": 2.33522629737854,
+      "learning_rate": 9.518372748330194e-05,
+      "loss": 2.3207,
+      "step": 194
+    },
+    {
+      "epoch": 0.3719165085388994,
+      "grad_norm": 2.3043212890625,
+      "learning_rate": 9.504844339512095e-05,
+      "loss": 2.2691,
+      "step": 196
+    },
+    {
+      "epoch": 0.3757115749525617,
+      "grad_norm": 2.3060500621795654,
+      "learning_rate": 9.4911384423061e-05,
+      "loss": 2.4741,
+      "step": 198
+    },
+    {
+      "epoch": 0.3795066413662239,
+      "grad_norm": 2.431614398956299,
+      "learning_rate": 9.477255596717012e-05,
+      "loss": 2.3706,
+      "step": 200
+    },
+    {
+      "epoch": 0.38330170777988615,
+      "grad_norm": 2.571744918823242,
+      "learning_rate": 9.463196349721308e-05,
+      "loss": 2.4345,
+      "step": 202
+    },
+    {
+      "epoch": 0.3870967741935484,
+      "grad_norm": 2.950385332107544,
+      "learning_rate": 9.448961255245584e-05,
+      "loss": 2.0978,
+      "step": 204
+    },
+    {
+      "epoch": 0.3908918406072106,
+      "grad_norm": 6.460769176483154,
+      "learning_rate": 9.434550874144728e-05,
+      "loss": 2.2981,
+      "step": 206
+    },
+    {
+      "epoch": 0.3946869070208729,
+      "grad_norm": 2.294574022293091,
+      "learning_rate": 9.419965774179824e-05,
+      "loss": 2.3483,
+      "step": 208
+    },
+    {
+      "epoch": 0.3984819734345351,
+      "grad_norm": 2.530029296875,
+      "learning_rate": 9.405206529995785e-05,
+      "loss": 2.4667,
+      "step": 210
+    },
+    {
+      "epoch": 0.40227703984819735,
+      "grad_norm": 2.0644922256469727,
+      "learning_rate": 9.39027372309871e-05,
+      "loss": 2.2927,
+      "step": 212
+    },
+    {
+      "epoch": 0.4060721062618596,
+      "grad_norm": 2.369694948196411,
+      "learning_rate": 9.375167941832973e-05,
+      "loss": 2.4687,
+      "step": 214
+    },
+    {
+      "epoch": 0.4098671726755218,
+      "grad_norm": 2.3800711631774902,
+      "learning_rate": 9.359889781358042e-05,
+      "loss": 2.3473,
+      "step": 216
+    },
+    {
+      "epoch": 0.41366223908918404,
+      "grad_norm": 2.2723309993743896,
+      "learning_rate": 9.344439843625034e-05,
+      "loss": 2.4378,
+      "step": 218
+    },
+    {
+      "epoch": 0.4174573055028463,
+      "grad_norm": 2.2413368225097656,
+      "learning_rate": 9.32881873735299e-05,
+      "loss": 2.391,
+      "step": 220
+    },
+    {
+      "epoch": 0.42125237191650855,
+      "grad_norm": 2.273754596710205,
+      "learning_rate": 9.313027078004903e-05,
+      "loss": 2.2899,
+      "step": 222
+    },
+    {
+      "epoch": 0.4250474383301708,
+      "grad_norm": 2.2374515533447266,
+      "learning_rate": 9.297065487763462e-05,
+      "loss": 2.3854,
+      "step": 224
+    },
+    {
+      "epoch": 0.428842504743833,
+      "grad_norm": 1.9647648334503174,
+      "learning_rate": 9.280934595506536e-05,
+      "loss": 2.1885,
+      "step": 226
+    },
+    {
+      "epoch": 0.43263757115749524,
+      "grad_norm": 2.4773480892181396,
+      "learning_rate": 9.264635036782405e-05,
+      "loss": 2.1969,
+      "step": 228
+    },
+    {
+      "epoch": 0.4364326375711575,
+      "grad_norm": 4.396137237548828,
+      "learning_rate": 9.248167453784711e-05,
+      "loss": 2.2705,
+      "step": 230
+    },
+    {
+      "epoch": 0.44022770398481975,
+      "grad_norm": 2.6246371269226074,
+      "learning_rate": 9.231532495327165e-05,
+      "loss": 2.3367,
+      "step": 232
+    },
+    {
+      "epoch": 0.444022770398482,
+      "grad_norm": 2.36479115486145,
+      "learning_rate": 9.21473081681797e-05,
+      "loss": 2.4254,
+      "step": 234
+    },
+    {
+      "epoch": 0.4478178368121442,
+      "grad_norm": 2.2640645503997803,
+      "learning_rate": 9.197763080234019e-05,
+      "loss": 2.198,
+      "step": 236
+    },
+    {
+      "epoch": 0.45161290322580644,
+      "grad_norm": 2.6508326530456543,
+      "learning_rate": 9.180629954094792e-05,
+      "loss": 2.253,
+      "step": 238
+    },
+    {
+      "epoch": 0.45540796963946867,
+      "grad_norm": 3.5617713928222656,
+      "learning_rate": 9.163332113436032e-05,
+      "loss": 2.1892,
+      "step": 240
+    },
+    {
+      "epoch": 0.45920303605313095,
+      "grad_norm": 2.4162395000457764,
+      "learning_rate": 9.145870239783142e-05,
+      "loss": 2.3496,
+      "step": 242
+    },
+    {
+      "epoch": 0.4629981024667932,
+      "grad_norm": 3.857107162475586,
+      "learning_rate": 9.128245021124334e-05,
+      "loss": 2.364,
+      "step": 244
+    },
+    {
+      "epoch": 0.4667931688804554,
+      "grad_norm": 2.012956142425537,
+      "learning_rate": 9.110457151883523e-05,
+      "loss": 2.2571,
+      "step": 246
+    },
+    {
+      "epoch": 0.47058823529411764,
+      "grad_norm": 1.9764139652252197,
+      "learning_rate": 9.092507332892968e-05,
+      "loss": 2.2254,
+      "step": 248
+    },
+    {
+      "epoch": 0.47438330170777987,
+      "grad_norm": 1.8924527168273926,
+      "learning_rate": 9.07439627136566e-05,
+      "loss": 2.1948,
+      "step": 250
+    },
+    {
+      "epoch": 0.4781783681214421,
+      "grad_norm": 2.961359739303589,
+      "learning_rate": 9.056124680867457e-05,
+      "loss": 2.1587,
+      "step": 252
+    },
+    {
+      "epoch": 0.4819734345351044,
+      "grad_norm": 2.010711193084717,
+      "learning_rate": 9.037693281288969e-05,
+      "loss": 2.3077,
+      "step": 254
+    },
+    {
+      "epoch": 0.4857685009487666,
+      "grad_norm": 2.0528528690338135,
+      "learning_rate": 9.019102798817197e-05,
+      "loss": 2.2348,
+      "step": 256
+    },
+    {
+      "epoch": 0.48956356736242884,
+      "grad_norm": 3.3071796894073486,
+      "learning_rate": 9.000353965906917e-05,
+      "loss": 2.3238,
+      "step": 258
+    },
+    {
+      "epoch": 0.49335863377609107,
+      "grad_norm": 2.2678682804107666,
+      "learning_rate": 8.981447521251831e-05,
+      "loss": 2.2368,
+      "step": 260
+    },
+    {
+      "epoch": 0.4971537001897533,
+      "grad_norm": 2.4855003356933594,
+      "learning_rate": 8.962384209755452e-05,
+      "loss": 2.0563,
+      "step": 262
+    },
+    {
+      "epoch": 0.5009487666034156,
+      "grad_norm": 2.1087934970855713,
+      "learning_rate": 8.943164782501765e-05,
+      "loss": 2.2727,
+      "step": 264
+    },
+    {
+      "epoch": 0.5047438330170778,
+      "grad_norm": 2.3885719776153564,
+      "learning_rate": 8.923789996725624e-05,
+      "loss": 2.2086,
+      "step": 266
+    },
+    {
+      "epoch": 0.50853889943074,
+      "grad_norm": 3.1673200130462646,
+      "learning_rate": 8.904260615782927e-05,
+      "loss": 2.1385,
+      "step": 268
+    },
+    {
+      "epoch": 0.5123339658444023,
+      "grad_norm": 2.06791353225708,
+      "learning_rate": 8.884577409120535e-05,
+      "loss": 2.2336,
+      "step": 270
+    },
+    {
+      "epoch": 0.5161290322580645,
+      "grad_norm": 2.3750381469726562,
+      "learning_rate": 8.864741152245963e-05,
+      "loss": 2.1569,
+      "step": 272
+    },
+    {
+      "epoch": 0.5199240986717267,
+      "grad_norm": 2.1411375999450684,
+      "learning_rate": 8.84475262669681e-05,
+      "loss": 2.2936,
+      "step": 274
+    },
+    {
+      "epoch": 0.523719165085389,
+      "grad_norm": 2.0463004112243652,
+      "learning_rate": 8.824612620009987e-05,
+      "loss": 2.3689,
+      "step": 276
+    },
+    {
+      "epoch": 0.5275142314990512,
+      "grad_norm": 1.7346696853637695,
+      "learning_rate": 8.804321925690672e-05,
+      "loss": 2.2445,
+      "step": 278
+    },
+    {
+      "epoch": 0.5313092979127134,
+      "grad_norm": 2.2764406204223633,
+      "learning_rate": 8.783881343181055e-05,
+      "loss": 2.1715,
+      "step": 280
+    },
+    {
+      "epoch": 0.5351043643263758,
+      "grad_norm": 2.2156970500946045,
+      "learning_rate": 8.763291677828838e-05,
+      "loss": 2.0616,
+      "step": 282
+    },
+    {
+      "epoch": 0.538899430740038,
+      "grad_norm": 2.32037091255188,
+      "learning_rate": 8.742553740855506e-05,
+      "loss": 2.2687,
+      "step": 284
+    },
+    {
+      "epoch": 0.5426944971537002,
+      "grad_norm": 1.864304542541504,
+      "learning_rate": 8.721668349324364e-05,
+      "loss": 2.2962,
+      "step": 286
+    },
+    {
+      "epoch": 0.5464895635673624,
+      "grad_norm": 2.056326389312744,
+      "learning_rate": 8.700636326108342e-05,
+      "loss": 2.3313,
+      "step": 288
+    },
+    {
+      "epoch": 0.5502846299810247,
+      "grad_norm": 2.4540798664093018,
+      "learning_rate": 8.679458499857582e-05,
+      "loss": 2.2259,
+      "step": 290
+    },
+    {
+      "epoch": 0.5540796963946869,
+      "grad_norm": 1.975327968597412,
+      "learning_rate": 8.658135704966786e-05,
+      "loss": 2.1908,
+      "step": 292
+    },
+    {
+      "epoch": 0.5578747628083491,
+      "grad_norm": 1.679593801498413,
+      "learning_rate": 8.636668781542336e-05,
+      "loss": 2.1169,
+      "step": 294
+    },
+    {
+      "epoch": 0.5616698292220114,
+      "grad_norm": 1.7750844955444336,
+      "learning_rate": 8.615058575369202e-05,
+      "loss": 2.0965,
+      "step": 296
+    },
+    {
+      "epoch": 0.5654648956356736,
+      "grad_norm": 1.9540330171585083,
+      "learning_rate": 8.593305937877614e-05,
+      "loss": 2.1185,
+      "step": 298
+    },
+    {
+      "epoch": 0.5692599620493358,
+      "grad_norm": 3.0039901733398438,
+      "learning_rate": 8.571411726109519e-05,
+      "loss": 2.2415,
+      "step": 300
+    },
+    {
+      "epoch": 0.573055028462998,
+      "grad_norm": 2.0792276859283447,
+      "learning_rate": 8.549376802684812e-05,
+      "loss": 2.2909,
+      "step": 302
+    },
+    {
+      "epoch": 0.5768500948766604,
+      "grad_norm": 1.9235445261001587,
+      "learning_rate": 8.527202035767349e-05,
+      "loss": 2.2691,
+      "step": 304
+    },
+    {
+      "epoch": 0.5806451612903226,
+      "grad_norm": 1.9486608505249023,
+      "learning_rate": 8.504888299030747e-05,
+      "loss": 2.141,
+      "step": 306
+    },
+    {
+      "epoch": 0.5844402277039848,
+      "grad_norm": 1.959185242652893,
+      "learning_rate": 8.482436471623951e-05,
+      "loss": 2.1965,
+      "step": 308
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 5.812629699707031,
+      "learning_rate": 8.459847438136605e-05,
+      "loss": 2.012,
+      "step": 310
+    },
+    {
+      "epoch": 0.5920303605313093,
+      "grad_norm": 1.9838637113571167,
+      "learning_rate": 8.437122088564198e-05,
+      "loss": 2.074,
+      "step": 312
+    },
+    {
+      "epoch": 0.5958254269449715,
+      "grad_norm": 1.9467284679412842,
+      "learning_rate": 8.414261318272996e-05,
+      "loss": 2.1882,
+      "step": 314
+    },
+    {
+      "epoch": 0.5996204933586338,
+      "grad_norm": 1.763068675994873,
+      "learning_rate": 8.391266027964771e-05,
+      "loss": 2.2197,
+      "step": 316
+    },
+    {
+      "epoch": 0.603415559772296,
+      "grad_norm": 2.123964309692383,
+      "learning_rate": 8.368137123641302e-05,
+      "loss": 2.1306,
+      "step": 318
+    },
+    {
+      "epoch": 0.6072106261859582,
+      "grad_norm": 1.9844577312469482,
+      "learning_rate": 8.344875516568695e-05,
+      "loss": 2.2215,
+      "step": 320
+    },
+    {
+      "epoch": 0.6110056925996205,
+      "grad_norm": 1.8439053297042847,
+      "learning_rate": 8.321482123241464e-05,
+      "loss": 2.0638,
+      "step": 322
+    },
+    {
+      "epoch": 0.6148007590132827,
+      "grad_norm": 1.7768611907958984,
+      "learning_rate": 8.297957865346437e-05,
+      "loss": 2.0954,
+      "step": 324
+    },
+    {
+      "epoch": 0.618595825426945,
+      "grad_norm": 2.3415918350219727,
+      "learning_rate": 8.274303669726426e-05,
+      "loss": 1.9216,
+      "step": 326
+    },
+    {
+      "epoch": 0.6223908918406073,
+      "grad_norm": 1.9908041954040527,
+      "learning_rate": 8.250520468343722e-05,
+      "loss": 2.1316,
+      "step": 328
+    },
+    {
+      "epoch": 0.6261859582542695,
+      "grad_norm": 1.989737868309021,
+      "learning_rate": 8.226609198243372e-05,
+      "loss": 2.1663,
+      "step": 330
+    },
+    {
+      "epoch": 0.6299810246679317,
+      "grad_norm": 2.194958209991455,
+      "learning_rate": 8.202570801516256e-05,
+      "loss": 2.0314,
+      "step": 332
+    },
+    {
+      "epoch": 0.6337760910815939,
+      "grad_norm": 1.9709926843643188,
+      "learning_rate": 8.178406225261981e-05,
+      "loss": 2.0792,
+      "step": 334
+    },
+    {
+      "epoch": 0.6375711574952562,
+      "grad_norm": 1.8470232486724854,
+      "learning_rate": 8.15411642155155e-05,
+      "loss": 2.0086,
+      "step": 336
+    },
+    {
+      "epoch": 0.6413662239089184,
+      "grad_norm": 1.961173415184021,
+      "learning_rate": 8.129702347389865e-05,
+      "loss": 2.1516,
+      "step": 338
+    },
+    {
+      "epoch": 0.6451612903225806,
+      "grad_norm": 1.9438867568969727,
+      "learning_rate": 8.105164964678009e-05,
+      "loss": 2.3398,
+      "step": 340
+    },
+    {
+      "epoch": 0.6489563567362429,
+      "grad_norm": 1.8040043115615845,
+      "learning_rate": 8.080505240175363e-05,
+      "loss": 2.1578,
+      "step": 342
+    },
+    {
+      "epoch": 0.6527514231499051,
+      "grad_norm": 2.0405383110046387,
+      "learning_rate": 8.055724145461495e-05,
+      "loss": 1.9948,
+      "step": 344
+    },
+    {
+      "epoch": 0.6565464895635673,
+      "grad_norm": 2.0209884643554688,
+      "learning_rate": 8.030822656897902e-05,
+      "loss": 2.1048,
+      "step": 346
+    },
+    {
+      "epoch": 0.6603415559772297,
+      "grad_norm": 1.9259743690490723,
+      "learning_rate": 8.005801755589532e-05,
+      "loss": 1.9457,
+      "step": 348
+    },
+    {
+      "epoch": 0.6641366223908919,
+      "grad_norm": 2.5305731296539307,
+      "learning_rate": 7.980662427346127e-05,
+      "loss": 1.8099,
+      "step": 350
+    },
+    {
+      "epoch": 0.6679316888045541,
+      "grad_norm": 2.0857901573181152,
+      "learning_rate": 7.955405662643384e-05,
+      "loss": 1.8904,
+      "step": 352
+    },
+    {
+      "epoch": 0.6717267552182163,
+      "grad_norm": 2.1652462482452393,
+      "learning_rate": 7.930032456583931e-05,
+      "loss": 2.0716,
+      "step": 354
+    },
+    {
+      "epoch": 0.6755218216318786,
+      "grad_norm": 2.2173564434051514,
+      "learning_rate": 7.904543808858127e-05,
+      "loss": 1.9861,
+      "step": 356
+    },
+    {
+      "epoch": 0.6793168880455408,
+      "grad_norm": 1.7087043523788452,
+      "learning_rate": 7.878940723704664e-05,
+      "loss": 1.9788,
+      "step": 358
+    },
+    {
+      "epoch": 0.683111954459203,
+      "grad_norm": 2.3170816898345947,
+      "learning_rate": 7.853224209871007e-05,
+      "loss": 2.0999,
+      "step": 360
+    },
+    {
+      "epoch": 0.6869070208728653,
+      "grad_norm": 1.8516716957092285,
+      "learning_rate": 7.82739528057365e-05,
+      "loss": 1.9476,
+      "step": 362
+    },
+    {
+      "epoch": 0.6907020872865275,
+      "grad_norm": 1.87705659866333,
+      "learning_rate": 7.801454953458193e-05,
+      "loss": 1.9741,
+      "step": 364
+    },
+    {
+      "epoch": 0.6944971537001897,
+      "grad_norm": 2.0102131366729736,
+      "learning_rate": 7.775404250559249e-05,
+      "loss": 2.2415,
+      "step": 366
+    },
+    {
+      "epoch": 0.698292220113852,
+      "grad_norm": 1.992819905281067,
+      "learning_rate": 7.749244198260175e-05,
+      "loss": 2.0201,
+      "step": 368
+    },
+    {
+      "epoch": 0.7020872865275142,
+      "grad_norm": 2.0950779914855957,
+      "learning_rate": 7.722975827252638e-05,
+      "loss": 2.2209,
+      "step": 370
+    },
+    {
+      "epoch": 0.7058823529411765,
+      "grad_norm": 7.29935884475708,
+      "learning_rate": 7.696600172495997e-05,
+      "loss": 2.0799,
+      "step": 372
+    },
+    {
+      "epoch": 0.7096774193548387,
+      "grad_norm": 1.9377464056015015,
+      "learning_rate": 7.670118273176534e-05,
+      "loss": 2.0681,
+      "step": 374
+    },
+    {
+      "epoch": 0.713472485768501,
+      "grad_norm": 2.465165853500366,
+      "learning_rate": 7.643531172666513e-05,
+      "loss": 2.0705,
+      "step": 376
+    },
+    {
+      "epoch": 0.7172675521821632,
+      "grad_norm": 1.7911758422851562,
+      "learning_rate": 7.616839918483061e-05,
+      "loss": 2.1271,
+      "step": 378
+    },
+    {
+      "epoch": 0.7210626185958254,
+      "grad_norm": 1.8003861904144287,
+      "learning_rate": 7.590045562246902e-05,
+      "loss": 2.0348,
+      "step": 380
+    },
+    {
+      "epoch": 0.7248576850094877,
+      "grad_norm": 1.9686284065246582,
+      "learning_rate": 7.563149159640929e-05,
+      "loss": 2.0137,
+      "step": 382
+    },
+    {
+      "epoch": 0.7286527514231499,
+      "grad_norm": 1.6355502605438232,
+      "learning_rate": 7.5361517703686e-05,
+      "loss": 1.978,
+      "step": 384
+    },
+    {
+      "epoch": 0.7324478178368121,
+      "grad_norm": 1.841895341873169,
+      "learning_rate": 7.509054458112202e-05,
+      "loss": 2.2376,
+      "step": 386
+    },
+    {
+      "epoch": 0.7362428842504743,
+      "grad_norm": 1.7870166301727295,
+      "learning_rate": 7.481858290490917e-05,
+      "loss": 1.9684,
+      "step": 388
+    },
+    {
+      "epoch": 0.7400379506641366,
+      "grad_norm": 1.8254350423812866,
+      "learning_rate": 7.45456433901879e-05,
+      "loss": 2.0589,
+      "step": 390
+    },
+    {
+      "epoch": 0.7438330170777988,
+      "grad_norm": 1.7817878723144531,
+      "learning_rate": 7.427173679062484e-05,
+      "loss": 2.0451,
+      "step": 392
+    },
+    {
+      "epoch": 0.7476280834914611,
+      "grad_norm": 1.9436228275299072,
+      "learning_rate": 7.399687389798933e-05,
+      "loss": 2.1209,
+      "step": 394
+    },
+    {
+      "epoch": 0.7514231499051234,
+      "grad_norm": 1.7727830410003662,
+      "learning_rate": 7.372106554172802e-05,
+      "loss": 1.997,
+      "step": 396
+    },
+    {
+      "epoch": 0.7552182163187856,
+      "grad_norm": 1.7208454608917236,
+      "learning_rate": 7.344432258853841e-05,
+      "loss": 2.0055,
+      "step": 398
+    },
+    {
+      "epoch": 0.7590132827324478,
+      "grad_norm": 1.7357947826385498,
+      "learning_rate": 7.316665594194053e-05,
+      "loss": 2.0916,
+      "step": 400
+    },
+    {
+      "epoch": 0.7628083491461101,
+      "grad_norm": 1.5990426540374756,
+      "learning_rate": 7.288807654184747e-05,
+      "loss": 2.0385,
+      "step": 402
+    },
+    {
+      "epoch": 0.7666034155597723,
+      "grad_norm": 1.6348865032196045,
+      "learning_rate": 7.260859536413429e-05,
+      "loss": 2.0467,
+      "step": 404
+    },
+    {
+      "epoch": 0.7703984819734345,
+      "grad_norm": 1.546189785003662,
+      "learning_rate": 7.232822342020557e-05,
+      "loss": 1.8996,
+      "step": 406
+    },
+    {
+      "epoch": 0.7741935483870968,
+      "grad_norm": 1.7460343837738037,
+      "learning_rate": 7.204697175656165e-05,
+      "loss": 2.1052,
+      "step": 408
+    },
+    {
+      "epoch": 0.777988614800759,
+      "grad_norm": 1.6906976699829102,
+      "learning_rate": 7.176485145436325e-05,
+      "loss": 2.0588,
+      "step": 410
+    },
+    {
+      "epoch": 0.7817836812144212,
+      "grad_norm": 1.9893949031829834,
+      "learning_rate": 7.148187362899505e-05,
+      "loss": 2.0776,
+      "step": 412
+    },
+    {
+      "epoch": 0.7855787476280834,
+      "grad_norm": 1.6658051013946533,
+      "learning_rate": 7.119804942962762e-05,
+      "loss": 1.8487,
+      "step": 414
+    },
+    {
+      "epoch": 0.7893738140417458,
+      "grad_norm": 1.7261987924575806,
+      "learning_rate": 7.091339003877826e-05,
+      "loss": 2.0895,
+      "step": 416
+    },
+    {
+      "epoch": 0.793168880455408,
+      "grad_norm": 1.6543688774108887,
+      "learning_rate": 7.062790667187029e-05,
+      "loss": 1.9453,
+      "step": 418
+    },
+    {
+      "epoch": 0.7969639468690702,
+      "grad_norm": 1.6983455419540405,
+      "learning_rate": 7.034161057679127e-05,
+      "loss": 1.9985,
+      "step": 420
+    },
+    {
+      "epoch": 0.8007590132827325,
+      "grad_norm": 1.6611037254333496,
+      "learning_rate": 7.005451303344979e-05,
+      "loss": 1.9633,
+      "step": 422
+    },
+    {
+      "epoch": 0.8045540796963947,
+      "grad_norm": 1.979058861732483,
+      "learning_rate": 6.976662535333107e-05,
+      "loss": 1.8833,
+      "step": 424
+    },
+    {
+      "epoch": 0.8083491461100569,
+      "grad_norm": 2.006376266479492,
+      "learning_rate": 6.947795887905127e-05,
+      "loss": 1.8475,
+      "step": 426
+    },
+    {
+      "epoch": 0.8121442125237192,
+      "grad_norm": 1.8918062448501587,
+      "learning_rate": 6.918852498391063e-05,
+      "loss": 1.9098,
+      "step": 428
+    },
+    {
+      "epoch": 0.8159392789373814,
+      "grad_norm": 1.933212161064148,
+      "learning_rate": 6.889833507144532e-05,
+      "loss": 2.1358,
+      "step": 430
+    },
+    {
+      "epoch": 0.8197343453510436,
+      "grad_norm": 1.8028526306152344,
+      "learning_rate": 6.860740057497823e-05,
+      "loss": 1.9799,
+      "step": 432
+    },
+    {
+      "epoch": 0.8235294117647058,
+      "grad_norm": 2.174152374267578,
+      "learning_rate": 6.831573295716837e-05,
+      "loss": 2.0668,
+      "step": 434
+    },
+    {
+      "epoch": 0.8273244781783681,
+      "grad_norm": 2.581892728805542,
+      "learning_rate": 6.802334370955941e-05,
+      "loss": 1.9118,
+      "step": 436
+    },
+    {
+      "epoch": 0.8311195445920304,
+      "grad_norm": 1.7758573293685913,
+      "learning_rate": 6.773024435212678e-05,
+      "loss": 2.0122,
+      "step": 438
+    },
+    {
+      "epoch": 0.8349146110056926,
+      "grad_norm": 1.7795305252075195,
+      "learning_rate": 6.743644643282388e-05,
+      "loss": 2.1017,
+      "step": 440
+    },
+    {
+      "epoch": 0.8387096774193549,
+      "grad_norm": 1.9165265560150146,
+      "learning_rate": 6.714196152712704e-05,
+      "loss": 2.1366,
+      "step": 442
+    },
+    {
+      "epoch": 0.8425047438330171,
+      "grad_norm": 3.430307149887085,
+      "learning_rate": 6.684680123757949e-05,
+      "loss": 1.9484,
+      "step": 444
+    },
+    {
+      "epoch": 0.8462998102466793,
+      "grad_norm": 1.6847901344299316,
+      "learning_rate": 6.65509771933342e-05,
+      "loss": 1.8189,
+      "step": 446
+    },
+    {
+      "epoch": 0.8500948766603416,
+      "grad_norm": 1.6464290618896484,
+      "learning_rate": 6.625450104969572e-05,
+      "loss": 1.9903,
+      "step": 448
+    },
+    {
+      "epoch": 0.8538899430740038,
+      "grad_norm": 1.7267745733261108,
+      "learning_rate": 6.595738448766095e-05,
+      "loss": 1.9767,
+      "step": 450
+    },
+    {
+      "epoch": 0.857685009487666,
+      "grad_norm": 1.9262585639953613,
+      "learning_rate": 6.565963921345895e-05,
+      "loss": 1.9669,
+      "step": 452
+    },
+    {
+      "epoch": 0.8614800759013282,
+      "grad_norm": 1.6643258333206177,
+      "learning_rate": 6.536127695808964e-05,
+      "loss": 1.9634,
+      "step": 454
+    },
+    {
+      "epoch": 0.8652751423149905,
+      "grad_norm": 1.577966332435608,
+      "learning_rate": 6.506230947686172e-05,
+      "loss": 1.8958,
+      "step": 456
+    },
+    {
+      "epoch": 0.8690702087286527,
+      "grad_norm": 2.840637683868408,
+      "learning_rate": 6.47627485489294e-05,
+      "loss": 2.0766,
+      "step": 458
+    },
+    {
+      "epoch": 0.872865275142315,
+      "grad_norm": 1.7307078838348389,
+      "learning_rate": 6.446260597682839e-05,
+      "loss": 1.846,
+      "step": 460
+    },
+    {
+      "epoch": 0.8766603415559773,
+      "grad_norm": 1.7440413236618042,
+      "learning_rate": 6.416189358601088e-05,
+      "loss": 1.8265,
+      "step": 462
+    },
+    {
+      "epoch": 0.8804554079696395,
+      "grad_norm": 1.5109995603561401,
+      "learning_rate": 6.386062322437954e-05,
+      "loss": 1.8226,
+      "step": 464
+    },
+    {
+      "epoch": 0.8842504743833017,
+      "grad_norm": 1.671194076538086,
+      "learning_rate": 6.355880676182086e-05,
+      "loss": 2.038,
+      "step": 466
+    },
+    {
+      "epoch": 0.888045540796964,
+      "grad_norm": 1.5358309745788574,
+      "learning_rate": 6.325645608973735e-05,
+      "loss": 1.9325,
+      "step": 468
+    },
+    {
+      "epoch": 0.8918406072106262,
+      "grad_norm": 1.6306613683700562,
+      "learning_rate": 6.295358312057914e-05,
+      "loss": 1.9589,
+      "step": 470
+    },
+    {
+      "epoch": 0.8956356736242884,
+      "grad_norm": 1.5553851127624512,
+      "learning_rate": 6.26501997873745e-05,
+      "loss": 1.8156,
+      "step": 472
+    },
+    {
+      "epoch": 0.8994307400379506,
+      "grad_norm": 1.6885318756103516,
+      "learning_rate": 6.234631804325981e-05,
+      "loss": 1.9792,
+      "step": 474
+    },
+    {
+      "epoch": 0.9032258064516129,
+      "grad_norm": 5.679869651794434,
+      "learning_rate": 6.204194986100857e-05,
+      "loss": 1.9165,
+      "step": 476
+    },
+    {
+      "epoch": 0.9070208728652751,
+      "grad_norm": 1.6526105403900146,
+      "learning_rate": 6.173710723255966e-05,
+      "loss": 2.0554,
+      "step": 478
+    },
+    {
+      "epoch": 0.9108159392789373,
+      "grad_norm": 1.577702283859253,
+      "learning_rate": 6.143180216854487e-05,
+      "loss": 1.7735,
+      "step": 480
+    },
+    {
+      "epoch": 0.9146110056925996,
+      "grad_norm": 2.24104380607605,
+      "learning_rate": 6.112604669781572e-05,
+      "loss": 1.9426,
+      "step": 482
+    },
+    {
+      "epoch": 0.9184060721062619,
+      "grad_norm": 1.5590384006500244,
+      "learning_rate": 6.081985286696949e-05,
+      "loss": 2.0698,
+      "step": 484
+    },
+    {
+      "epoch": 0.9222011385199241,
+      "grad_norm": 1.7752505540847778,
+      "learning_rate": 6.051323273987463e-05,
+      "loss": 1.889,
+      "step": 486
+    },
+    {
+      "epoch": 0.9259962049335864,
+      "grad_norm": 1.7600066661834717,
+      "learning_rate": 6.020619839719538e-05,
+      "loss": 1.9693,
+      "step": 488
+    },
+    {
+      "epoch": 0.9297912713472486,
+      "grad_norm": 1.5522412061691284,
+      "learning_rate": 5.989876193591589e-05,
+      "loss": 1.9813,
+      "step": 490
+    },
+    {
+      "epoch": 0.9335863377609108,
+      "grad_norm": 2.223928213119507,
+      "learning_rate": 5.959093546886356e-05,
+      "loss": 1.954,
+      "step": 492
+    },
+    {
+      "epoch": 0.937381404174573,
+      "grad_norm": 2.466945171356201,
+      "learning_rate": 5.928273112423177e-05,
+      "loss": 1.7863,
+      "step": 494
+    },
+    {
+      "epoch": 0.9411764705882353,
+      "grad_norm": 1.743293285369873,
+      "learning_rate": 5.897416104510211e-05,
+      "loss": 1.9432,
+      "step": 496
+    },
+    {
+      "epoch": 0.9449715370018975,
+      "grad_norm": 1.6011027097702026,
+      "learning_rate": 5.866523738896587e-05,
+      "loss": 1.8718,
+      "step": 498
+    },
+    {
+      "epoch": 0.9487666034155597,
+      "grad_norm": 1.6478995084762573,
+      "learning_rate": 5.835597232724511e-05,
+      "loss": 1.9411,
+      "step": 500
+    },
+    {
+      "epoch": 0.952561669829222,
+      "grad_norm": 1.6733437776565552,
+      "learning_rate": 5.804637804481306e-05,
+      "loss": 1.8868,
+      "step": 502
+    },
+    {
+      "epoch": 0.9563567362428842,
+      "grad_norm": 1.6197586059570312,
+      "learning_rate": 5.773646673951406e-05,
+      "loss": 1.9805,
+      "step": 504
+    },
+    {
+      "epoch": 0.9601518026565465,
+      "grad_norm": 1.7262201309204102,
+      "learning_rate": 5.742625062168303e-05,
+      "loss": 2.0499,
+      "step": 506
+    },
+    {
+      "epoch": 0.9639468690702088,
+      "grad_norm": 1.5373488664627075,
+      "learning_rate": 5.7115741913664264e-05,
+      "loss": 1.8859,
+      "step": 508
+    },
+    {
+      "epoch": 0.967741935483871,
+      "grad_norm": 2.0669445991516113,
+      "learning_rate": 5.680495284933e-05,
+      "loss": 1.9428,
+      "step": 510
+    },
+    {
+      "epoch": 0.9715370018975332,
+      "grad_norm": 2.279287099838257,
+      "learning_rate": 5.649389567359831e-05,
+      "loss": 1.9444,
+      "step": 512
+    },
+    {
+      "epoch": 0.9753320683111955,
+      "grad_norm": 1.7030115127563477,
+      "learning_rate": 5.6182582641950764e-05,
+      "loss": 1.7837,
+      "step": 514
+    },
+    {
+      "epoch": 0.9791271347248577,
+      "grad_norm": 1.6407768726348877,
+      "learning_rate": 5.58710260199495e-05,
+      "loss": 1.8909,
+      "step": 516
+    },
+    {
+      "epoch": 0.9829222011385199,
+      "grad_norm": 1.5273981094360352,
+      "learning_rate": 5.555923808275395e-05,
+      "loss": 1.736,
+      "step": 518
+    },
+    {
+      "epoch": 0.9867172675521821,
+      "grad_norm": 1.762691617012024,
+      "learning_rate": 5.5247231114637256e-05,
+      "loss": 1.7998,
+      "step": 520
+    },
+    {
+      "epoch": 0.9905123339658444,
+      "grad_norm": 1.6185078620910645,
+      "learning_rate": 5.4935017408502274e-05,
+      "loss": 1.9654,
+      "step": 522
+    },
+    {
+      "epoch": 0.9943074003795066,
+      "grad_norm": 1.666943907737732,
+      "learning_rate": 5.462260926539722e-05,
+      "loss": 1.9379,
+      "step": 524
+    },
+    {
+      "epoch": 0.9981024667931688,
+      "grad_norm": 1.5249921083450317,
+      "learning_rate": 5.431001899403098e-05,
+      "loss": 1.9387,
+      "step": 526
+    }
+  ],
+  "logging_steps": 2,
+  "max_steps": 1054,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 527,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.912185436248474e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-527/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20386650688791ad0c29b49ae8cdcf0798f5d0868194e8c0a536314658aafe5a
+size 5432

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/OLMo-2-1124-7B-mni_Beng-coli-rand/",
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100384
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 100257,
+  "pad_token_id": 100277,
+  "transformers_version": "4.48.0.dev0"
+}

model-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d1480335fcfd79184fe631e1b0aa77a59ac348fa5ac329223dcdf46a1bd81bd
+size 4950070472

model-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0b6a357d03676c4986588aaf4dc96afc3f44a0498514961d2b2cfa1940655e
+size 4991622816

model-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1038a7d346a681a8fb6949e054c2246f3f3c80e826855c1cec6af1155cc607d
+size 4924546896

model-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c0a21146ace02d36928787ae7f34be868f7c81b894dea98ec7b0e5a3ea01b2
+size 4857404928

model-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f82a3014ccfc0f67208f63785ae4bc71f09aba86db48bf6139d18178abbfed
+size 4857404928

model-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a028ac98f31b7584432073229033962115043c4b055b0381c6d75b53761827f3
+size 4614509080

model.safetensors.index.json

@@ -0,0 +1,362 @@
+{
+  "metadata": {
+    "total_size": 29195517952
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,189 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20386650688791ad0c29b49ae8cdcf0798f5d0868194e8c0a536314658aafe5a
+size 5432