Upload OpenLMForCausalLM

config.json

@@ -0,0 +1,28 @@
+{
+  "apply_qk_norm": true,
+  "architectures": [
+    "OpenLMForCausalLM"
+  ],
+  "attn_func": "xformers_attn",
+  "dim": 96,
+  "ffn_type": "swiglu",
+  "model_type": "openlm",
+  "moe_capacity_factor": 1.25,
+  "moe_expert_model_parallelism": false,
+  "moe_freq": 0,
+  "moe_loss_weight": 0.1,
+  "moe_num_experts": null,
+  "moe_top_k": 2,
+  "moe_weight_parallelism": false,
+  "n_heads": 4,
+  "n_layers": 3,
+  "norm_eps": 1e-05,
+  "norm_type": "default_layer_norm",
+  "positional_embedding_type": "rotary",
+  "post_embed_norm": false,
+  "seq_len": 2048,
+  "torch_dtype": "float32",
+  "transformers_version": "4.49.0",
+  "vocab_size": 50432,
+  "weight_tying": false
+}

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.49.0"
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76626e270d0022ae7460e9139710fbe53a8ca8e782150259515b4fb098c913f8
-size 40073056
+oid sha256:401458d637dbf8cb62d142cb435b51ea22f18bf513ec4db324309ed6b744e67d
+size 40073344