nick11roberts
/

resolving_test

model_hub_mixin

pytorch_model_hub_mixin

Model card Files Files and versions

nick11roberts commited on Jul 14

Commit

5dcf1d0

·

verified ·

1 Parent(s): e3cfea0

Upload OpenLMForCausalLM

Files changed (2) hide show

config.json +2 -2
model.safetensors +2 -2

config.json CHANGED Viewed

@@ -5,7 +5,7 @@
   ],
   "attn_func": "xformers_attn",
   "attn_name": "xformers_attn",
-  "dim": 1504,
   "ffn_type": "swiglu",
   "model_type": "openlm",
   "moe_capacity_factor": 1.25,
@@ -16,7 +16,7 @@
   "moe_top_k": 2,
   "moe_weight_parallelism": false,
   "n_heads": 4,
-  "n_layers": 30,
   "norm_eps": 1e-05,
   "norm_type": "default_layer_norm",
   "positional_embedding_type": "rotary",

   ],
   "attn_func": "xformers_attn",
   "attn_name": "xformers_attn",
+  "dim": 96,
   "ffn_type": "swiglu",
   "model_type": "openlm",
   "moe_capacity_factor": 1.25,
   "moe_top_k": 2,
   "moe_weight_parallelism": false,
   "n_heads": 4,
+  "n_layers": 3,
   "norm_eps": 1e-05,
   "norm_type": "default_layer_norm",
   "positional_embedding_type": "rotary",

model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4ffe924a768c8caee3edcd631ee3dbd70abe67bfc46c8954ca0c0430a8a3c39
-size 3911825832

 version https://git-lfs.github.com/spec/v1
+oid sha256:401458d637dbf8cb62d142cb435b51ea22f18bf513ec4db324309ed6b744e67d
+size 40073344