initial

config.json

@@ -0,0 +1 @@
+{"name": "LayeredMambaLM", "input": {"vocab_size": 128256, "tie_embeddings": true, "pad_vocab_size_multiple": 8, "lm_head_bias": false}, "MixerModel": {"name": "MixerModel", "input": {"d_model": 3072, "n_layer": 28, "lm_head_prenorm": "rms"}, "Blocks": [{"name": "LlamaBlock", "n_layers": 28, "input": {"resid_dropout": 0.0, "mlp_intermediate_size": 8192, "mlp_act_fn": "silu"}, "Layer": {"name": "DiscreteMamba2", "input": {"d_state": 64, "n_qk_heads": 32, "n_v_heads": 32, "expand": 1, "chunk_size": 128, "activation": "identity", "use_ref_impl": false, "bias": false, "norm_cls": "none", "initializer": {"a_log": "default", "x": "default", "B": "default", "C": "default", "D": "default", "z": "identity", "out": "default", "convolution": "identity"}}}}]}}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:505fe96aa4466bcb51d8fa250f5a814549b2c2f39d0e83968d2b476c3ac1d798
+size 14628013984