initial

config.json

@@ -0,0 +1 @@
+{"name": "LayeredMambaLM", "input": {"vocab_size": 128256, "tie_embeddings": true, "pad_vocab_size_multiple": 8, "lm_head_bias": false}, "MixerModel": {"name": "MixerModel", "input": {"d_model": 4096, "n_layer": 32, "lm_head_prenorm": "rms"}, "Blocks": [{"name": "LlamaBlock", "n_layers": 32, "input": {"resid_dropout": 0.0, "mlp_intermediate_size": 14336, "mlp_act_fn": "silu"}, "Layer": {"name": "DiscreteMamba2", "input": {"d_state": 64, "n_qk_heads": 32, "n_v_heads": 32, "expand": 1, "chunk_size": 128, "activation": "identity", "use_ref_impl": false, "bias": false, "norm_cls": "none", "initializer": {"a_log": "default", "x": "default", "B": "default", "C": "default", "D": "default", "z": "identity", "out": "default", "convolution": "identity"}}}}]}}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1117435f6d445fe75876768d81f6d1206bf32e5e938ed9920c88827e963c651e
+size 33263513264