(Trained with Unsloth)

config.json

@@ -12,14 +12,17 @@
   "hidden_size": 2304,
   "initializer_range": 0.1,
   "intermediate_size": 5760,
-  "max_position_embeddings": 8192,
+  "max_position_embeddings": 32768,
   "model_type": "llama",
   "num_attention_heads": 36,
   "num_hidden_layers": 40,
   "num_key_value_heads": 36,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
+  "rope_scaling": {
+    "factor": 4.0,
+    "type": "linear"
+  },
   "rope_theta": 10000.0,
   "scale_depth": 1.4,
   "scale_emb": 12,

pytorch_model-00001-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e6306fd324d43c93a09f1af05b48424827f8c0b72f55723639b68d1f7d698aa
+oid sha256:a51b0ca88bab3fca5379779ce51e5633dfdff89eb38f2e63b1be56f13129b7dd
 size 4993313262

pytorch_model-00002-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9fb7d5b0f57a8936b837e343d19f2334649b102619c5744c887eb570315cce7
+oid sha256:1629609cd855d782e8c7a5fd96f28aadc7191d342ff9a64c4743fe3d3ef42ce1
 size 1022223648

tokenizer_config.json

@@ -38,7 +38,7 @@
     "input_ids",
     "attention_mask"
   ],
-  "model_max_length": 8192,
+  "model_max_length": 32768,
   "pad_token": "</s>",
   "padding_side": "right",
   "sep_token": null,