{
    "LanguageModel":{
       "type":"LayeredMambaLM",
       "input":{
          "vocab_size":51200,
          "pad_vocab_size_multiple":8
       }
    },
    "MixerModel":{
       "type":"LayerMixerModel",
       "input":{
          "d_model":2048,
          "n_layer":24,
          "lm_head_prenorm":"layer"
       }
    },
    "Block1":{
       "n_layers":4,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.discrete_mamba2",
       "core_input":{
          "d_state":64,
          "n_v_heads":32,
          "n_qk_heads":32,
          "d_conv":4,
          "conv_bias":true,
          "expand":1,
          "chunk_size":128,
          "activation":"identity",
          "bias":false
       }
    },
    "Block2":{
       "n_layers":1,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.phi_attention",
       "core_input":{
          
       }
    },
    "Block3":{
       "n_layers":1,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.discrete_mamba2",
       "core_input":{
          "d_state":64,
          "n_v_heads":32,
          "n_qk_heads":32,
          "d_conv":4,
          "conv_bias":true,
          "expand":1,
          "chunk_size":128,
          "activation":"identity",
          "bias":false
       }
    },
    "Block4":{
       "n_layers":4,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.discrete_mamba2",
       "core_input":{
          "d_state":64,
          "nheads":32,
          "expand":1,
          "activation":"identity",
          "use_ref_impl":false,
          "bias":false,
          "norm_cls":"none"
       }
    },
    "Block5":{
       "n_layers":1,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.phi_attention",
       "core_input":{
          
       }
    },
    "Block6":{
       "n_layers":1,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.discrete_mamba2",
       "core_input":{
          "d_state":64,
          "n_v_heads":32,
          "n_qk_heads":32,
          "d_conv":4,
          "conv_bias":true,
          "expand":1,
          "chunk_size":128,
          "activation":"identity",
          "bias":false
       }
    },
    "Block7":{
       "n_layers":4,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.discrete_mamba2",
       "core_input":{
          "d_state":64,
          "n_v_heads":32,
          "n_qk_heads":32,
          "d_conv":4,
          "conv_bias":true,
          "expand":1,
          "chunk_size":128,
          "activation":"identity",
          "bias":false
       }
    },
    "Block8":{
       "n_layers":1,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.phi_attention",
       "core_input":{
          
       }
    },
    "Block9":{
       "n_layers":1,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.discrete_mamba2",
       "core_input":{
          "d_state":64,
          "n_v_heads":32,
          "n_qk_heads":32,
          "d_conv":4,
          "conv_bias":true,
          "expand":1,
          "chunk_size":128,
          "activation":"identity",
          "bias":false
       }
    },
    "Block10":{
       "n_layers":4,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.discrete_mamba2",
       "core_input":{
          "d_state":64,
          "n_v_heads":32,
          "n_qk_heads":32,
          "d_conv":4,
          "conv_bias":true,
          "expand":1,
          "chunk_size":128,
          "activation":"identity",
          "bias":false
       }
    },
    "Block11":{
       "n_layers":1,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.phi_attention",
       "core_input":{
          
       }
    },
    "Block12":{
       "n_layers":1,
       "BlockType":"modules.phi_block",
       "block_input":{
          "resid_dropout":0.0
       },
       "CoreType":"modules.mixers.discrete_mamba2",
       "core_input":{
          "d_state":64,
          "n_v_heads":32,
          "n_qk_heads":32,
          "d_conv":4,
          "conv_bias":true,
          "expand":1,
          "chunk_size":128,
          "activation":"identity",
          "bias":false
       }
    }
 }