Added small model with RoPE and MHLA

README.md

@@ -2,10 +2,6 @@
 tags:
 - model_hub_mixin
 - pytorch_model_hub_mixin
-datasets:
-- IgorVolochay/russian_jokes
-language:
-- ru
 ---
 
 This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:

config.json

@@ -1,10 +1,10 @@
 {
   "dropout": 0.1,
-  "hidden_dim": 96,
-  "intermediate_dim": 256,
+  "hidden_dim": 768,
+  "intermediate_dim": 3072,
   "max_seq_len": 128,
-  "n_head": 4,
-  "n_kv_head": 2,
-  "n_layer": 3,
+  "n_head": 12,
+  "n_kv_head": 12,
+  "n_layer": 12,
   "vocab_size": 1024
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17701fdda46da06d5f7d042a73f410914a526191769ca3f74f0edb85f903eb17
-size 1925360
+oid sha256:d4eccc9d9d3b8b35ca109573b0a8c96b23cd077f942bc6b20515bb06267855a1
+size 460642712