Upload folder using huggingface_hub

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "BioMistral/BioMistral-7B",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 3,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

mergekit_moe_config.yml

@@ -0,0 +1,13 @@
+base_model: BioMistral/BioMistral-7B
+gate_mode: hidden # one of "hidden", "cheap_embed", or "random"
+dtype: bfloat16 # output dtype (float32, float16, or bfloat16)
+experts:
+  - source_model: Treza12/BioMistral-invasive-FULL
+    positive_prompts:
+      - "This is a prompt that is demonstrative excels at invasive Breast Cancer"
+  - source_model: Treza12/BioMistral-non-invasive-FULL
+    positive_prompts:
+      - "This is a prompt that is demonstrative excels at non invasive benign Breast Cancer"
+  - source_model: Treza12/BioMistral-advanced-FULL
+    positive_prompts:
+      - "This is a prompt that is demonstrative excels at advanced Stage Breast Cancer"

model-00001-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:566dad9786518d3172b04d2e6f11ae353a4444fb3b23ecd314d67441d03bc322
+size 933250496

model-00002-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cce693f1e45b79564d64cb906469d9c896ebea24dc8a32f3bbf51bdf0eeeaafe
+size 905987672

model-00003-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:913206795918e6e7ca26c65c4697e518933170a3f9f1e4448bbeadfd5206ce80
+size 905987672

model-00004-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18d0b12ff7f6756d2724cbc45e2af4746e3d6adf08f2a36c74b4e9634103b82a
+size 989865456

model-00005-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc8aee0a7aac84ca33d2e971257d7a3ce650be3a9b09e848e06131ad060f8587
+size 973088024

model-00006-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff67a3bf065a3785491f7864e5b291ac7d55bfc935088b3ec77e7823b42c3e61
+size 905987672

model-00007-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2474bd036afb2ac05da225462361b11e38030431e626e1e5b6e11e3fbf7ffbd
+size 905987672

model-00008-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30b6ce62dff53e8133b487eb65024bfa2e102052c4a337939e77c9e411c0bcb2
+size 905987672

model-00009-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63ea6a93a83f17663bd4923f145cd9a3c66483a4ad3f9210f12ef7c8a4d98a4c
+size 905987672

model-00010-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02d35b0faf11bdaec0661b41c7f014e5c94cee7b7e7b9cd736d3391475f2ccb4
+size 939525160

model-00011-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d9de9c5ecba1de7542ea6e731d44f90375d625c80a21099c38d948440d0b00f
+size 905987672

model-00012-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb3cac9ee3dfac04a42aa180d4c8d914107f6e1f8d806d85cbad581f4f498af8
+size 905987672

model-00013-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4639587c27dc07cf22a5de030b49b17afa9bc0083a41e5f6432bf815ac63db91
+size 905987680

model-00014-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db77ad1732a9b04c4eacaad9c0d2750fd6b4260f1222c50dfa4457d59ddfc1ed
+size 905987688

model-00015-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eea3400deb437248864c518f1b91f8a228c62b2f5580872a225af17de2833111
+size 939525168

model-00016-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1bc20635bb3df3ff6b2bebd59be8485b5f4eab8e7b85583e34b6b0df17caaa3
+size 905987688

model-00017-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b1fdccb335f41af4500bf45ac7d551f1bde84983705f114241a21d1e43770cd
+size 905987688

model-00018-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:331a1f01534499cde2292c84652bbcc40e7dc22d9e2aaf0af830aef5b12c294a
+size 905987688

model-00019-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a8ebcda57bc66e782ecb00e3ecf60ee447b659826aa72181b1a8894fe55e9e4
+size 905987688

model-00020-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e362074848d513206dc32d48a19f9fa1464381b06e2bf8c0002d39ba63382855
+size 939525168

model-00021-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65891be1dc6c17a3e1d036e717cb8c70aae22d766c97f13d968624aff5a9a8c0
+size 905987688

model-00022-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23aee1a1cccc1ab215e9c3eb087c269e896a5965b94b865ea2d16ac03f2f7c64
+size 905987688

model-00023-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82238adb23148bf4eedfcd5b50995eb2efd5806aa20c764a1196311f8eed25af
+size 905987688

model-00024-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6684ecfabb2b616ce0a388f96d393ad080ae3e7b8da5097e7e3386c1ef77b5cb
+size 905987688

model-00025-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a03890cc2ad76993f575648ba13932cae497b98d6c834e5afe0ca5fb8c12b86
+size 939525168

model-00026-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4558fced0bb2b482ffe06e1a80185062357a6beeadbffbc9090e32da04e6e76d
+size 905987688

model-00027-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47766b4dd3f5d62ff3396353680234a9e69091ac94424660d0fd186ccf6abce7
+size 905987688

model-00028-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad07d978fa9e324b002f342615978e218cbb172118c3d89bb22544ed4608e3d2
+size 905987688

model-00029-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7456411277b1455c30aeea41a7c4693d807a1c5ff52d055f2e37ce72761e99ee
+size 905987688

model-00030-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a7c56fcb08387c7b987e91965697a24b6b8503ab9d23f488351e71ead00170a
+size 939525168

model-00031-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76aa0270318ec746a5b1f35ecdaf9e5503486d32caba6f18b4eceafcab40b85e
+size 905987688

model-00032-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f152316bff1b008b0257e5764aff59fa53142473c58005aaa68ce8aaae5062d3
+size 905987688

model-00033-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:672551ec3b2b3f23b752cb65f43737ba3dd8757ae0ac2bbd2e7acb963c3e5fa7
+size 905987688

model-00034-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58046fa81d4b0076168cd49f615acee24cd0e77662f4be60e8dbfe4a439f2264
+size 905987688

model-00035-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79299cca532fa1e8e2d2b1dd36344db09160dc0922b3d56a9d86723c5eba6939
+size 939525168

model-00036-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e174cf2a9ab527e9705243830e1bdafef4b63501f1ca1c511977a4f77b088e28
+size 905987688

model-00037-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c72c46f5c4199b7ceab14092eb010bb4ee45553ce985e5c25edc2fb2e40ffa10
+size 905987688

model-00038-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:321508f32e879887db825ebf9483ba3a18462b09dbdaa86deccdc15199b4e8be
+size 905987688

model-00039-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c7d58872ce75fb4cea3f3957bc50330fe570eade99b127f2adfe90dd78f0e2a
+size 905987688

model-00040-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:675bf373d853d1d826fc9f69aca9a04d6c7bfe5cff33748ed36dd9d478efbc16
+size 939525168

model-00041-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3c63b8b99576f13168b2d7e2eb0c33c4d54c6206996334186a765b4582923ba
+size 380383248

model.safetensors.index.json

@@ -0,0 +1 @@
+{"metadata": {"mergekit_version": "0.0.4.4", "total_size": 37032828928}, "weight_map": {"model.embed_tokens.weight": "model-00001-of-00041.safetensors", "model.layers.0.input_layernorm.weight": "model-00001-of-00041.safetensors", "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00041.safetensors", "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00041.safetensors", "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00041.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00041.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00041.safetensors", "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00041.safetensors", "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00041.safetensors", "model.layers.0.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00041.safetensors", "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00041.safetensors", "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00041.safetensors", "model.layers.0.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00041.safetensors", "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00041.safetensors", "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00041.safetensors", "model.layers.0.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00041.safetensors", "model.layers.1.input_layernorm.weight": "model-00002-of-00041.safetensors", "model.layers.1.self_attn.q_proj.weight": "model-00002-of-00041.safetensors", "model.layers.1.self_attn.k_proj.weight": "model-00002-of-00041.safetensors", "model.layers.1.self_attn.v_proj.weight": "model-00002-of-00041.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00002-of-00041.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00041.safetensors", "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00041.safetensors", "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00041.safetensors", "model.layers.1.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00041.safetensors", "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00041.safetensors", "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00041.safetensors", "model.layers.1.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00041.safetensors", "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00041.safetensors", "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00041.safetensors", "model.layers.1.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00041.safetensors", "model.layers.2.input_layernorm.weight": "model-00003-of-00041.safetensors", "model.layers.2.self_attn.q_proj.weight": "model-00003-of-00041.safetensors", "model.layers.2.self_attn.k_proj.weight": "model-00003-of-00041.safetensors", "model.layers.2.self_attn.v_proj.weight": "model-00003-of-00041.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00003-of-00041.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00003-of-00041.safetensors", "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00041.safetensors", "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00041.safetensors", "model.layers.2.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00041.safetensors", "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00041.safetensors", "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00041.safetensors", "model.layers.2.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00041.safetensors", "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00041.safetensors", "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00041.safetensors", "model.layers.2.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00041.safetensors", "model.layers.3.input_layernorm.weight": "model-00004-of-00041.safetensors", "model.layers.3.self_attn.q_proj.weight": "model-00004-of-00041.safetensors", "model.layers.3.self_attn.k_proj.weight": "model-00004-of-00041.safetensors", "model.layers.3.self_attn.v_proj.weight": "model-00004-of-00041.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00005-of-00041.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00005-of-00041.safetensors", "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00041.safetensors", "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00041.safetensors", "model.layers.3.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00041.safetensors", "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00041.safetensors", "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00041.safetensors", "model.layers.3.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00041.safetensors", "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00041.safetensors", "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00041.safetensors", "model.layers.3.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00041.safetensors", "model.layers.4.input_layernorm.weight": "model-00006-of-00041.safetensors", "model.layers.4.self_attn.q_proj.weight": "model-00006-of-00041.safetensors", "model.layers.4.self_attn.k_proj.weight": "model-00006-of-00041.safetensors", "model.layers.4.self_attn.v_proj.weight": "model-00006-of-00041.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00006-of-00041.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00006-of-00041.safetensors", "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00041.safetensors", "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00041.safetensors", "model.layers.4.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00041.safetensors", "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00041.safetensors", "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00041.safetensors", "model.layers.4.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00041.safetensors", "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00041.safetensors", "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00041.safetensors", "model.layers.4.block_sparse_moe.experts.2.w2.weight": "model-00007-of-00041.safetensors", "model.layers.5.input_layernorm.weight": "model-00007-of-00041.safetensors", "model.layers.5.self_attn.q_proj.weight": "model-00007-of-00041.safetensors", "model.layers.5.self_attn.k_proj.weight": "model-00007-of-00041.safetensors", "model.layers.5.self_attn.v_proj.weight": "model-00007-of-00041.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00007-of-00041.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00007-of-00041.safetensors", "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00007-of-00041.safetensors", "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00007-of-00041.safetensors", "model.layers.5.block_sparse_moe.experts.2.w3.weight": "model-00007-of-00041.safetensors", "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00007-of-00041.safetensors", "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00008-of-00041.safetensors", "model.layers.5.block_sparse_moe.experts.2.w1.weight": "model-00008-of-00041.safetensors", "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00041.safetensors", "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00041.safetensors", "model.layers.5.block_sparse_moe.experts.2.w2.weight": "model-00008-of-00041.safetensors", "model.layers.6.input_layernorm.weight": "model-00008-of-00041.safetensors", "model.layers.6.self_attn.q_proj.weight": "model-00008-of-00041.safetensors", "model.layers.6.self_attn.k_proj.weight": "model-00008-of-00041.safetensors", "model.layers.6.self_attn.v_proj.weight": "model-00008-of-00041.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00008-of-00041.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00008-of-00041.safetensors", "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00008-of-00041.safetensors", "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00008-of-00041.safetensors", "model.layers.6.block_sparse_moe.experts.2.w3.weight": "model-00009-of-00041.safetensors", "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00041.safetensors", "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00041.safetensors", "model.layers.6.block_sparse_moe.experts.2.w1.weight": "model-00009-of-00041.safetensors", "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00009-of-00041.safetensors", "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00009-of-00041.safetensors", "model.layers.6.block_sparse_moe.experts.2.w2.weight": "model-00009-of-00041.safetensors", "model.layers.7.input_layernorm.weight": "model-00009-of-00041.safetensors", "model.layers.7.self_attn.q_proj.weight": "model-00009-of-00041.safetensors", "model.layers.7.self_attn.k_proj.weight": "model-00009-of-00041.safetensors", "model.layers.7.self_attn.v_proj.weight": "model-00009-of-00041.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00009-of-00041.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00009-of-00041.safetensors", "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00010-of-00041.safetensors", "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00010-of-00041.safetensors", "model.layers.7.block_sparse_moe.experts.2.w3.weight": "model-00010-of-00041.safetensors", "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00041.safetensors", "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00041.safetensors", "model.layers.7.block_sparse_moe.experts.2.w1.weight": "model-00010-of-00041.safetensors", "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00010-of-00041.safetensors", "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00010-of-00041.safetensors", "model.layers.7.block_sparse_moe.experts.2.w2.weight": "model-00011-of-00041.safetensors", "model.layers.8.input_layernorm.weight": "model-00011-of-00041.safetensors", "model.layers.8.self_attn.q_proj.weight": "model-00011-of-00041.safetensors", "model.layers.8.self_attn.k_proj.weight": "model-00011-of-00041.safetensors", "model.layers.8.self_attn.v_proj.weight": "model-00011-of-00041.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00011-of-00041.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00011-of-00041.safetensors", "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00011-of-00041.safetensors", "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00011-of-00041.safetensors", "model.layers.8.block_sparse_moe.experts.2.w3.weight": "model-00011-of-00041.safetensors", "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00041.safetensors", "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00041.safetensors", "model.layers.8.block_sparse_moe.experts.2.w1.weight": "model-00011-of-00041.safetensors", "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00012-of-00041.safetensors", "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00012-of-00041.safetensors", "model.layers.8.block_sparse_moe.experts.2.w2.weight": "model-00012-of-00041.safetensors", "model.layers.9.input_layernorm.weight": "model-00012-of-00041.safetensors", "model.layers.9.self_attn.q_proj.weight": "model-00012-of-00041.safetensors", "model.layers.9.self_attn.k_proj.weight": "model-00012-of-00041.safetensors", "model.layers.9.self_attn.v_proj.weight": "model-00012-of-00041.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00012-of-00041.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00012-of-00041.safetensors", "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00012-of-00041.safetensors", "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00012-of-00041.safetensors", "model.layers.9.block_sparse_moe.experts.2.w3.weight": "model-00012-of-00041.safetensors", "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00041.safetensors", "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00013-of-00041.safetensors", "model.layers.9.block_sparse_moe.experts.2.w1.weight": "model-00013-of-00041.safetensors", "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00013-of-00041.safetensors", "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00013-of-00041.safetensors", "model.layers.9.block_sparse_moe.experts.2.w2.weight": "model-00013-of-00041.safetensors", "model.layers.10.input_layernorm.weight": "model-00013-of-00041.safetensors", "model.layers.10.self_attn.q_proj.weight": "model-00013-of-00041.safetensors", "model.layers.10.self_attn.k_proj.weight": "model-00013-of-00041.safetensors", "model.layers.10.self_attn.v_proj.weight": "model-00013-of-00041.safetensors", "model.layers.10.self_attn.o_proj.weight": "model-00013-of-00041.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00013-of-00041.safetensors", "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00013-of-00041.safetensors", "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00013-of-00041.safetensors", "model.layers.10.block_sparse_moe.experts.2.w3.weight": "model-00014-of-00041.safetensors", "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00014-of-00041.safetensors", "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00014-of-00041.safetensors", "model.layers.10.block_sparse_moe.experts.2.w1.weight": "model-00014-of-00041.safetensors", "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00014-of-00041.safetensors", "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00014-of-00041.safetensors", "model.layers.10.block_sparse_moe.experts.2.w2.weight": "model-00014-of-00041.safetensors", "model.layers.11.input_layernorm.weight": "model-00014-of-00041.safetensors", "model.layers.11.self_attn.q_proj.weight": "model-00014-of-00041.safetensors", "model.layers.11.self_attn.k_proj.weight": "model-00014-of-00041.safetensors", "model.layers.11.self_attn.v_proj.weight": "model-00014-of-00041.safetensors", "model.layers.11.self_attn.o_proj.weight": "model-00014-of-00041.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00014-of-00041.safetensors", "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00015-of-00041.safetensors", "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00015-of-00041.safetensors", "model.layers.11.block_sparse_moe.experts.2.w3.weight": "model-00015-of-00041.safetensors", "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00015-of-00041.safetensors", "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00015-of-00041.safetensors", "model.layers.11.block_sparse_moe.experts.2.w1.weight": "model-00015-of-00041.safetensors", "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00015-of-00041.safetensors", "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00015-of-00041.safetensors", "model.layers.11.block_sparse_moe.experts.2.w2.weight": "model-00016-of-00041.safetensors", "model.layers.12.input_layernorm.weight": "model-00016-of-00041.safetensors", "model.layers.12.self_attn.q_proj.weight": "model-00016-of-00041.safetensors", "model.layers.12.self_attn.k_proj.weight": "model-00016-of-00041.safetensors", "model.layers.12.self_attn.v_proj.weight": "model-00016-of-00041.safetensors", "model.layers.12.self_attn.o_proj.weight": "model-00016-of-00041.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00016-of-00041.safetensors", "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00016-of-00041.safetensors", "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00016-of-00041.safetensors", "model.layers.12.block_sparse_moe.experts.2.w3.weight": "model-00016-of-00041.safetensors", "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00016-of-00041.safetensors", "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00016-of-00041.safetensors", "model.layers.12.block_sparse_moe.experts.2.w1.weight": "model-00016-of-00041.safetensors", "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00017-of-00041.safetensors", "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00017-of-00041.safetensors", "model.layers.12.block_sparse_moe.experts.2.w2.weight": "model-00017-of-00041.safetensors", "model.layers.13.input_layernorm.weight": "model-00017-of-00041.safetensors", "model.layers.13.self_attn.q_proj.weight": "model-00017-of-00041.safetensors", "model.layers.13.self_attn.k_proj.weight": "model-00017-of-00041.safetensors", "model.layers.13.self_attn.v_proj.weight": "model-00017-of-00041.safetensors", "model.layers.13.self_attn.o_proj.weight": "model-00017-of-00041.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00017-of-00041.safetensors", "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00017-of-00041.safetensors", "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00017-of-00041.safetensors", "model.layers.13.block_sparse_moe.experts.2.w3.weight": "model-00017-of-00041.safetensors", "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00017-of-00041.safetensors", "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00018-of-00041.safetensors", "model.layers.13.block_sparse_moe.experts.2.w1.weight": "model-00018-of-00041.safetensors", "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00018-of-00041.safetensors", "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00018-of-00041.safetensors", "model.layers.13.block_sparse_moe.experts.2.w2.weight": "model-00018-of-00041.safetensors", "model.layers.14.input_layernorm.weight": "model-00018-of-00041.safetensors", "model.layers.14.self_attn.q_proj.weight": "model-00018-of-00041.safetensors", "model.layers.14.self_attn.k_proj.weight": "model-00018-of-00041.safetensors", "model.layers.14.self_attn.v_proj.weight": "model-00018-of-00041.safetensors", "model.layers.14.self_attn.o_proj.weight": "model-00018-of-00041.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00018-of-00041.safetensors", "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00018-of-00041.safetensors", "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00018-of-00041.safetensors", "model.layers.14.block_sparse_moe.experts.2.w3.weight": "model-00019-of-00041.safetensors", "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00019-of-00041.safetensors", "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00019-of-00041.safetensors", "model.layers.14.block_sparse_moe.experts.2.w1.weight": "model-00019-of-00041.safetensors", "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00019-of-00041.safetensors", "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00019-of-00041.safetensors", "model.layers.14.block_sparse_moe.experts.2.w2.weight": "model-00019-of-00041.safetensors", "model.layers.15.input_layernorm.weight": "model-00019-of-00041.safetensors", "model.layers.15.self_attn.q_proj.weight": "model-00019-of-00041.safetensors", "model.layers.15.self_attn.k_proj.weight": "model-00019-of-00041.safetensors", "model.layers.15.self_attn.v_proj.weight": "model-00019-of-00041.safetensors", "model.layers.15.self_attn.o_proj.weight": "model-00019-of-00041.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00019-of-00041.safetensors", "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00020-of-00041.safetensors", "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00020-of-00041.safetensors", "model.layers.15.block_sparse_moe.experts.2.w3.weight": "model-00020-of-00041.safetensors", "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00020-of-00041.safetensors", "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00020-of-00041.safetensors", "model.layers.15.block_sparse_moe.experts.2.w1.weight": "model-00020-of-00041.safetensors", "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00020-of-00041.safetensors", "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00020-of-00041.safetensors", "model.layers.15.block_sparse_moe.experts.2.w2.weight": "model-00021-of-00041.safetensors", "model.layers.16.input_layernorm.weight": "model-00021-of-00041.safetensors", "model.layers.16.self_attn.q_proj.weight": "model-00021-of-00041.safetensors", "model.layers.16.self_attn.k_proj.weight": "model-00021-of-00041.safetensors", "model.layers.16.self_attn.v_proj.weight": "model-00021-of-00041.safetensors", "model.layers.16.self_attn.o_proj.weight": "model-00021-of-00041.safetensors", "model.layers.16.post_attention_layernorm.weight": "model-00021-of-00041.safetensors", "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00021-of-00041.safetensors", "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00021-of-00041.safetensors", "model.layers.16.block_sparse_moe.experts.2.w3.weight": "model-00021-of-00041.safetensors", "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00021-of-00041.safetensors", "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00021-of-00041.safetensors", "model.layers.16.block_sparse_moe.experts.2.w1.weight": "model-00021-of-00041.safetensors", "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00022-of-00041.safetensors", "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00022-of-00041.safetensors", "model.layers.16.block_sparse_moe.experts.2.w2.weight": "model-00022-of-00041.safetensors", "model.layers.17.input_layernorm.weight": "model-00022-of-00041.safetensors", "model.layers.17.self_attn.q_proj.weight": "model-00022-of-00041.safetensors", "model.layers.17.self_attn.k_proj.weight": "model-00022-of-00041.safetensors", "model.layers.17.self_attn.v_proj.weight": "model-00022-of-00041.safetensors", "model.layers.17.self_attn.o_proj.weight": "model-00022-of-00041.safetensors", "model.layers.17.post_attention_layernorm.weight": "model-00022-of-00041.safetensors", "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00022-of-00041.safetensors", "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00022-of-00041.safetensors", "model.layers.17.block_sparse_moe.experts.2.w3.weight": "model-00022-of-00041.safetensors", "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00022-of-00041.safetensors", "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00023-of-00041.safetensors", "model.layers.17.block_sparse_moe.experts.2.w1.weight": "model-00023-of-00041.safetensors", "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00023-of-00041.safetensors", "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00023-of-00041.safetensors", "model.layers.17.block_sparse_moe.experts.2.w2.weight": "model-00023-of-00041.safetensors", "model.layers.18.input_layernorm.weight": "model-00023-of-00041.safetensors", "model.layers.18.self_attn.q_proj.weight": "model-00023-of-00041.safetensors", "model.layers.18.self_attn.k_proj.weight": "model-00023-of-00041.safetensors", "model.layers.18.self_attn.v_proj.weight": "model-00023-of-00041.safetensors", "model.layers.18.self_attn.o_proj.weight": "model-00023-of-00041.safetensors", "model.layers.18.post_attention_layernorm.weight": "model-00023-of-00041.safetensors", "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00023-of-00041.safetensors", "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00023-of-00041.safetensors", "model.layers.18.block_sparse_moe.experts.2.w3.weight": "model-00024-of-00041.safetensors", "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00024-of-00041.safetensors", "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00024-of-00041.safetensors", "model.layers.18.block_sparse_moe.experts.2.w1.weight": "model-00024-of-00041.safetensors", "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00024-of-00041.safetensors", "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00024-of-00041.safetensors", "model.layers.18.block_sparse_moe.experts.2.w2.weight": "model-00024-of-00041.safetensors", "model.layers.19.input_layernorm.weight": "model-00024-of-00041.safetensors", "model.layers.19.self_attn.q_proj.weight": "model-00024-of-00041.safetensors", "model.layers.19.self_attn.k_proj.weight": "model-00024-of-00041.safetensors", "model.layers.19.self_attn.v_proj.weight": "model-00024-of-00041.safetensors", "model.layers.19.self_attn.o_proj.weight": "model-00024-of-00041.safetensors", "model.layers.19.post_attention_layernorm.weight": "model-00024-of-00041.safetensors", "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00025-of-00041.safetensors", "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00025-of-00041.safetensors", "model.layers.19.block_sparse_moe.experts.2.w3.weight": "model-00025-of-00041.safetensors", "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00025-of-00041.safetensors", "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00025-of-00041.safetensors", "model.layers.19.block_sparse_moe.experts.2.w1.weight": "model-00025-of-00041.safetensors", "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00025-of-00041.safetensors", "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00025-of-00041.safetensors", "model.layers.19.block_sparse_moe.experts.2.w2.weight": "model-00026-of-00041.safetensors", "model.layers.20.input_layernorm.weight": "model-00026-of-00041.safetensors", "model.layers.20.self_attn.q_proj.weight": "model-00026-of-00041.safetensors", "model.layers.20.self_attn.k_proj.weight": "model-00026-of-00041.safetensors", "model.layers.20.self_attn.v_proj.weight": "model-00026-of-00041.safetensors", "model.layers.20.self_attn.o_proj.weight": "model-00026-of-00041.safetensors", "model.layers.20.post_attention_layernorm.weight": "model-00026-of-00041.safetensors", "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00026-of-00041.safetensors", "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00026-of-00041.safetensors", "model.layers.20.block_sparse_moe.experts.2.w3.weight": "model-00026-of-00041.safetensors", "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00026-of-00041.safetensors", "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00026-of-00041.safetensors", "model.layers.20.block_sparse_moe.experts.2.w1.weight": "model-00026-of-00041.safetensors", "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00027-of-00041.safetensors", "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00027-of-00041.safetensors", "model.layers.20.block_sparse_moe.experts.2.w2.weight": "model-00027-of-00041.safetensors", "model.layers.21.input_layernorm.weight": "model-00027-of-00041.safetensors", "model.layers.21.self_attn.q_proj.weight": "model-00027-of-00041.safetensors", "model.layers.21.self_attn.k_proj.weight": "model-00027-of-00041.safetensors", "model.layers.21.self_attn.v_proj.weight": "model-00027-of-00041.safetensors", "model.layers.21.self_attn.o_proj.weight": "model-00027-of-00041.safetensors", "model.layers.21.post_attention_layernorm.weight": "model-00027-of-00041.safetensors", "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00027-of-00041.safetensors", "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00027-of-00041.safetensors", "model.layers.21.block_sparse_moe.experts.2.w3.weight": "model-00027-of-00041.safetensors", "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00027-of-00041.safetensors", "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00028-of-00041.safetensors", "model.layers.21.block_sparse_moe.experts.2.w1.weight": "model-00028-of-00041.safetensors", "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00028-of-00041.safetensors", "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00028-of-00041.safetensors", "model.layers.21.block_sparse_moe.experts.2.w2.weight": "model-00028-of-00041.safetensors", "model.layers.22.input_layernorm.weight": "model-00028-of-00041.safetensors", "model.layers.22.self_attn.q_proj.weight": "model-00028-of-00041.safetensors", "model.layers.22.self_attn.k_proj.weight": "model-00028-of-00041.safetensors", "model.layers.22.self_attn.v_proj.weight": "model-00028-of-00041.safetensors", "model.layers.22.self_attn.o_proj.weight": "model-00028-of-00041.safetensors", "model.layers.22.post_attention_layernorm.weight": "model-00028-of-00041.safetensors", "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00028-of-00041.safetensors", "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00028-of-00041.safetensors", "model.layers.22.block_sparse_moe.experts.2.w3.weight": "model-00029-of-00041.safetensors", "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00029-of-00041.safetensors", "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00029-of-00041.safetensors", "model.layers.22.block_sparse_moe.experts.2.w1.weight": "model-00029-of-00041.safetensors", "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00029-of-00041.safetensors", "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00029-of-00041.safetensors", "model.layers.22.block_sparse_moe.experts.2.w2.weight": "model-00029-of-00041.safetensors", "model.layers.23.input_layernorm.weight": "model-00029-of-00041.safetensors", "model.layers.23.self_attn.q_proj.weight": "model-00029-of-00041.safetensors", "model.layers.23.self_attn.k_proj.weight": "model-00029-of-00041.safetensors", "model.layers.23.self_attn.v_proj.weight": "model-00029-of-00041.safetensors", "model.layers.23.self_attn.o_proj.weight": "model-00029-of-00041.safetensors", "model.layers.23.post_attention_layernorm.weight": "model-00029-of-00041.safetensors", "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00030-of-00041.safetensors", "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00030-of-00041.safetensors", "model.layers.23.block_sparse_moe.experts.2.w3.weight": "model-00030-of-00041.safetensors", "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00030-of-00041.safetensors", "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00030-of-00041.safetensors", "model.layers.23.block_sparse_moe.experts.2.w1.weight": "model-00030-of-00041.safetensors", "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00030-of-00041.safetensors", "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00030-of-00041.safetensors", "model.layers.23.block_sparse_moe.experts.2.w2.weight": "model-00031-of-00041.safetensors", "model.layers.24.input_layernorm.weight": "model-00031-of-00041.safetensors", "model.layers.24.self_attn.q_proj.weight": "model-00031-of-00041.safetensors", "model.layers.24.self_attn.k_proj.weight": "model-00031-of-00041.safetensors", "model.layers.24.self_attn.v_proj.weight": "model-00031-of-00041.safetensors", "model.layers.24.self_attn.o_proj.weight": "model-00031-of-00041.safetensors", "model.layers.24.post_attention_layernorm.weight": "model-00031-of-00041.safetensors", "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00031-of-00041.safetensors", "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00031-of-00041.safetensors", "model.layers.24.block_sparse_moe.experts.2.w3.weight": "model-00031-of-00041.safetensors", "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00031-of-00041.safetensors", "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00031-of-00041.safetensors", "model.layers.24.block_sparse_moe.experts.2.w1.weight": "model-00031-of-00041.safetensors", "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00032-of-00041.safetensors", "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00032-of-00041.safetensors", "model.layers.24.block_sparse_moe.experts.2.w2.weight": "model-00032-of-00041.safetensors", "model.layers.25.input_layernorm.weight": "model-00032-of-00041.safetensors", "model.layers.25.self_attn.q_proj.weight": "model-00032-of-00041.safetensors", "model.layers.25.self_attn.k_proj.weight": "model-00032-of-00041.safetensors", "model.layers.25.self_attn.v_proj.weight": "model-00032-of-00041.safetensors", "model.layers.25.self_attn.o_proj.weight": "model-00032-of-00041.safetensors", "model.layers.25.post_attention_layernorm.weight": "model-00032-of-00041.safetensors", "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00032-of-00041.safetensors", "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00032-of-00041.safetensors", "model.layers.25.block_sparse_moe.experts.2.w3.weight": "model-00032-of-00041.safetensors", "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00032-of-00041.safetensors", "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00033-of-00041.safetensors", "model.layers.25.block_sparse_moe.experts.2.w1.weight": "model-00033-of-00041.safetensors", "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00033-of-00041.safetensors", "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00033-of-00041.safetensors", "model.layers.25.block_sparse_moe.experts.2.w2.weight": "model-00033-of-00041.safetensors", "model.layers.26.input_layernorm.weight": "model-00033-of-00041.safetensors", "model.layers.26.self_attn.q_proj.weight": "model-00033-of-00041.safetensors", "model.layers.26.self_attn.k_proj.weight": "model-00033-of-00041.safetensors", "model.layers.26.self_attn.v_proj.weight": "model-00033-of-00041.safetensors", "model.layers.26.self_attn.o_proj.weight": "model-00033-of-00041.safetensors", "model.layers.26.post_attention_layernorm.weight": "model-00033-of-00041.safetensors", "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00033-of-00041.safetensors", "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00033-of-00041.safetensors", "model.layers.26.block_sparse_moe.experts.2.w3.weight": "model-00034-of-00041.safetensors", "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00034-of-00041.safetensors", "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00034-of-00041.safetensors", "model.layers.26.block_sparse_moe.experts.2.w1.weight": "model-00034-of-00041.safetensors", "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00034-of-00041.safetensors", "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00034-of-00041.safetensors", "model.layers.26.block_sparse_moe.experts.2.w2.weight": "model-00034-of-00041.safetensors", "model.layers.27.input_layernorm.weight": "model-00034-of-00041.safetensors", "model.layers.27.self_attn.q_proj.weight": "model-00034-of-00041.safetensors", "model.layers.27.self_attn.k_proj.weight": "model-00034-of-00041.safetensors", "model.layers.27.self_attn.v_proj.weight": "model-00034-of-00041.safetensors", "model.layers.27.self_attn.o_proj.weight": "model-00034-of-00041.safetensors", "model.layers.27.post_attention_layernorm.weight": "model-00034-of-00041.safetensors", "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00035-of-00041.safetensors", "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00035-of-00041.safetensors", "model.layers.27.block_sparse_moe.experts.2.w3.weight": "model-00035-of-00041.safetensors", "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00035-of-00041.safetensors", "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00035-of-00041.safetensors", "model.layers.27.block_sparse_moe.experts.2.w1.weight": "model-00035-of-00041.safetensors", "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00035-of-00041.safetensors", "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00035-of-00041.safetensors", "model.layers.27.block_sparse_moe.experts.2.w2.weight": "model-00036-of-00041.safetensors", "model.layers.28.input_layernorm.weight": "model-00036-of-00041.safetensors", "model.layers.28.self_attn.q_proj.weight": "model-00036-of-00041.safetensors", "model.layers.28.self_attn.k_proj.weight": "model-00036-of-00041.safetensors", "model.layers.28.self_attn.v_proj.weight": "model-00036-of-00041.safetensors", "model.layers.28.self_attn.o_proj.weight": "model-00036-of-00041.safetensors", "model.layers.28.post_attention_layernorm.weight": "model-00036-of-00041.safetensors", "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00036-of-00041.safetensors", "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00036-of-00041.safetensors", "model.layers.28.block_sparse_moe.experts.2.w3.weight": "model-00036-of-00041.safetensors", "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00036-of-00041.safetensors", "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00036-of-00041.safetensors", "model.layers.28.block_sparse_moe.experts.2.w1.weight": "model-00036-of-00041.safetensors", "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00037-of-00041.safetensors", "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00037-of-00041.safetensors", "model.layers.28.block_sparse_moe.experts.2.w2.weight": "model-00037-of-00041.safetensors", "model.layers.29.input_layernorm.weight": "model-00037-of-00041.safetensors", "model.layers.29.self_attn.q_proj.weight": "model-00037-of-00041.safetensors", "model.layers.29.self_attn.k_proj.weight": "model-00037-of-00041.safetensors", "model.layers.29.self_attn.v_proj.weight": "model-00037-of-00041.safetensors", "model.layers.29.self_attn.o_proj.weight": "model-00037-of-00041.safetensors", "model.layers.29.post_attention_layernorm.weight": "model-00037-of-00041.safetensors", "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00037-of-00041.safetensors", "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00037-of-00041.safetensors", "model.layers.29.block_sparse_moe.experts.2.w3.weight": "model-00037-of-00041.safetensors", "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00037-of-00041.safetensors", "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00038-of-00041.safetensors", "model.layers.29.block_sparse_moe.experts.2.w1.weight": "model-00038-of-00041.safetensors", "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00038-of-00041.safetensors", "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00038-of-00041.safetensors", "model.layers.29.block_sparse_moe.experts.2.w2.weight": "model-00038-of-00041.safetensors", "model.layers.30.input_layernorm.weight": "model-00038-of-00041.safetensors", "model.layers.30.self_attn.q_proj.weight": "model-00038-of-00041.safetensors", "model.layers.30.self_attn.k_proj.weight": "model-00038-of-00041.safetensors", "model.layers.30.self_attn.v_proj.weight": "model-00038-of-00041.safetensors", "model.layers.30.self_attn.o_proj.weight": "model-00038-of-00041.safetensors", "model.layers.30.post_attention_layernorm.weight": "model-00038-of-00041.safetensors", "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00038-of-00041.safetensors", "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00038-of-00041.safetensors", "model.layers.30.block_sparse_moe.experts.2.w3.weight": "model-00039-of-00041.safetensors", "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00039-of-00041.safetensors", "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00039-of-00041.safetensors", "model.layers.30.block_sparse_moe.experts.2.w1.weight": "model-00039-of-00041.safetensors", "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00039-of-00041.safetensors", "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00039-of-00041.safetensors", "model.layers.30.block_sparse_moe.experts.2.w2.weight": "model-00039-of-00041.safetensors", "model.layers.31.input_layernorm.weight": "model-00039-of-00041.safetensors", "model.layers.31.self_attn.q_proj.weight": "model-00039-of-00041.safetensors", "model.layers.31.self_attn.k_proj.weight": "model-00039-of-00041.safetensors", "model.layers.31.self_attn.v_proj.weight": "model-00039-of-00041.safetensors", "model.layers.31.self_attn.o_proj.weight": "model-00039-of-00041.safetensors", "model.layers.31.post_attention_layernorm.weight": "model-00039-of-00041.safetensors", "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00040-of-00041.safetensors", "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00040-of-00041.safetensors", "model.layers.31.block_sparse_moe.experts.2.w3.weight": "model-00040-of-00041.safetensors", "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00040-of-00041.safetensors", "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00040-of-00041.safetensors", "model.layers.31.block_sparse_moe.experts.2.w1.weight": "model-00040-of-00041.safetensors", "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00040-of-00041.safetensors", "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00040-of-00041.safetensors", "model.layers.31.block_sparse_moe.experts.2.w2.weight": "model-00041-of-00041.safetensors", "model.norm.weight": "model-00041-of-00041.safetensors", "lm_head.weight": "model-00041-of-00041.safetensors", "model.layers.0.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.1.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.2.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.3.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.4.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.5.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.6.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.7.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.8.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.9.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.10.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.11.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.12.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.13.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.14.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.15.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.16.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.17.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.18.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.19.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.20.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.21.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.22.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.23.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.24.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.25.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.26.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.27.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.28.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.29.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.30.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors", "model.layers.31.block_sparse_moe.gate.weight": "model-00041-of-00041.safetensors"}}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}