Upload folder using huggingface_hub

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "deepseek-ai/deepseek-llm-7b-base",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "num_local_experts": 3,
+  "output_router_logits": false,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": true,
+  "vocab_size": 102400
+}

mergekit_moe_config.yml

@@ -0,0 +1,13 @@
+base_model: deepseek-ai/deepseek-llm-7b-base
+gate_mode: hidden # one of "hidden", "cheap_embed", or "random"
+dtype: bfloat16 # output dtype (float32, float16, or bfloat16)
+experts:
+  - source_model: Treza12/DeepSeek-invasive-FULL
+    positive_prompts:
+      - "This is a prompt that is demonstrative excels at invasive Breast Cancer"
+  - source_model: Treza12/non_invasive_benign-FULL
+    positive_prompts:
+      - "This is a prompt that is demonstrative excels at non invasive benign Breast Cancer"
+  - source_model: Treza12/DeepSeek-advanced_stage
+    positive_prompts:
+      - "This is a prompt that is demonstrative excels at advanced Stage Breast Cancer"

model-00001-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d48effd9bd53e8a53118f69b577c27771964cabb1343425e80b15cce83ec20c8
+size 973095736

model-00002-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49ccc151699eb4fb7fd067b41d79090664b1302a58c0aba23aaa0f6206c31b10
+size 945833816

model-00003-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:736b1f0a2dd56f48a22dca5892ccf7d9cc042fdf26f68ccc6b35a04173bf533b
+size 945833816

model-00004-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1b0bdaf857e84ee481b36e278c008da9e4c82a5e4a032c03d7838bc4f02d35e
+size 945833816

model-00005-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3cfc085986d7791e205f7da61f024a5c3df4da00e5df39940377b0b17428968
+size 945833816

model-00006-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8858c8c69f2612174c48ae9db2db3b520c21cfebe5c56bbd9893f25a6ee334b8
+size 945833816

model-00007-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a30050d66ad0a1976825e22b711bc9f8898bdb677fe4761797618b5dc7e8ab4
+size 945833816

model-00008-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a9b3dd5575c93caf6adaa107504d5859ac0dc8cd2cfef79bfea92d11026bc12
+size 945833816

model-00009-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d825ed00efbf2e013470cf3f418e8ce0590ef5beeb20b4f99038546fcbb7f6c8
+size 945833816

model-00010-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:039bb0a729d107a95f05af247bf52dc735f524dd3c92dd33761ace666ddfd5b6
+size 945833816

model-00011-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:031e53c1ea7a411451e18c870225379b1964c4ac54f65b147d1454ad0add8caa
+size 945833800

model-00012-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10bbc5ed1b3919a04ae93c0eb3447a74b405750c0542cec87b7d0b4c52765848
+size 945833832

model-00013-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cf2d0e07ccf856a4d91a766a17f0d9f5eadaaa20c428b663c16091292616f4d
+size 945833832

model-00014-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0db0f60db935849169e8b86a46f357e3b9d66d5234cfc7048748c3c9493ecf46
+size 945833832

model-00015-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01ccfb44c2b758e854095dcf783f5730124fac8b663d6d0096f6148e4488f8c5
+size 945833832

model-00016-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc0a736e8693eba3fc4378bcdf1d09c58bc44d7900322c7d9bbc9b69bd6a7ff1
+size 945833832

model-00017-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d7d9060e54a695e4d563684edfb255a7221aab58f29db2451ad9175e747540e
+size 945833832

model-00018-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7093e4cbd37a1d0b27f331ebf2d9e8f3ec1c7133f82e237a042c1f18578f2ad6
+size 945833832

model-00019-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e96406d43393500af546abcabaa87a8fed82e2dda54ae73a5e8345cbba4dd4da
+size 945833832

model-00020-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70bf81e1443629894808dd8422b3fc1c2599d5bbfadb050f46f7bee002a21e62
+size 945833832

model-00021-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:befa33b5b044062dabf9076eb76bb9b829172f8d6053679bd9cfa163b52cfe4b
+size 945833832

model-00022-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa86dec4d1e87f958a630d4af619f2128f39162a9013c17d956e8e3aeb22ecfa
+size 945833832

model-00023-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9374f0e2d52453864c8080987ad8bc242d16e5b839d2d3946a33a24bf0767e7f
+size 945833832

model-00024-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:226c17e871cd3b325449cfdf823dcf4894b739bf6c6d4188513d2ecd7a518f0f
+size 945833832

model-00025-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27d2b4fa946710cc74b2888febd197210be3d10dd7ab5ca4bc470c4bd76f2652
+size 945833832

model-00026-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8339df51fc84e30e92ff89d80b46d7f03f573232a06bcc591baae37578cfebc6
+size 945833832

model-00027-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e64163262df7d06de36a2a4bba1fed915f2eefd8ef594624cb34d16cda1c087c
+size 945833832

model-00028-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f532f2f72d8add92c1ae4a48c2e2ec9bf15f0b452b3fbb2c8a2cc30fdc59fdd6
+size 945833832

model-00029-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92db1b28702e322d7cc7d8423116e8beb1654d782ad90207c740bac672fded0a
+size 945833832

model-00030-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d99a9f53882dc94014d93608011c509ba51162def8445577ec9399bb42e96ce3
+size 945833832

model-00031-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58844ab3b371001a38c80c63b791a95f520c310cc8d53703fd744b97a5675456
+size 811607312

model-00032-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4be4095d548e91bea48ae93b80b5d95772df9d03e93f99eee6d599ed27e2220f
+size 839601736

model.safetensors.index.json

@@ -0,0 +1 @@
+{"metadata": {"mergekit_version": "0.0.4.4", "total_size": 30053425152}, "weight_map": {"model.embed_tokens.weight": "model-00001-of-00032.safetensors", "model.layers.0.input_layernorm.weight": "model-00001-of-00032.safetensors", "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00032.safetensors", "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00032.safetensors", "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00032.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00032.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00032.safetensors", "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00032.safetensors", "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00032.safetensors", "model.layers.0.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00032.safetensors", "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00032.safetensors", "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00032.safetensors", "model.layers.0.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00032.safetensors", "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00032.safetensors", "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00032.safetensors", "model.layers.0.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00032.safetensors", "model.layers.1.input_layernorm.weight": "model-00002-of-00032.safetensors", "model.layers.1.self_attn.q_proj.weight": "model-00002-of-00032.safetensors", "model.layers.1.self_attn.k_proj.weight": "model-00002-of-00032.safetensors", "model.layers.1.self_attn.v_proj.weight": "model-00002-of-00032.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00002-of-00032.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00032.safetensors", "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00032.safetensors", "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00032.safetensors", "model.layers.1.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00032.safetensors", "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00032.safetensors", "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00032.safetensors", "model.layers.1.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00032.safetensors", "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00032.safetensors", "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00032.safetensors", "model.layers.1.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00032.safetensors", "model.layers.2.input_layernorm.weight": "model-00003-of-00032.safetensors", "model.layers.2.self_attn.q_proj.weight": "model-00003-of-00032.safetensors", "model.layers.2.self_attn.k_proj.weight": "model-00003-of-00032.safetensors", "model.layers.2.self_attn.v_proj.weight": "model-00003-of-00032.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00003-of-00032.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00003-of-00032.safetensors", "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00032.safetensors", "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00032.safetensors", "model.layers.2.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00032.safetensors", "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00032.safetensors", "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00032.safetensors", "model.layers.2.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00032.safetensors", "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00032.safetensors", "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00032.safetensors", "model.layers.2.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00032.safetensors", "model.layers.3.input_layernorm.weight": "model-00004-of-00032.safetensors", "model.layers.3.self_attn.q_proj.weight": "model-00004-of-00032.safetensors", "model.layers.3.self_attn.k_proj.weight": "model-00004-of-00032.safetensors", "model.layers.3.self_attn.v_proj.weight": "model-00004-of-00032.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00004-of-00032.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00004-of-00032.safetensors", "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00032.safetensors", "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00032.safetensors", "model.layers.3.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00032.safetensors", "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00032.safetensors", "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00032.safetensors", "model.layers.3.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00032.safetensors", "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00032.safetensors", "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00032.safetensors", "model.layers.3.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00032.safetensors", "model.layers.4.input_layernorm.weight": "model-00005-of-00032.safetensors", "model.layers.4.self_attn.q_proj.weight": "model-00005-of-00032.safetensors", "model.layers.4.self_attn.k_proj.weight": "model-00005-of-00032.safetensors", "model.layers.4.self_attn.v_proj.weight": "model-00005-of-00032.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00005-of-00032.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00005-of-00032.safetensors", "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00032.safetensors", "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00032.safetensors", "model.layers.4.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00032.safetensors", "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00032.safetensors", "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00032.safetensors", "model.layers.4.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00032.safetensors", "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00032.safetensors", "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00032.safetensors", "model.layers.4.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00032.safetensors", "model.layers.5.input_layernorm.weight": "model-00006-of-00032.safetensors", "model.layers.5.self_attn.q_proj.weight": "model-00006-of-00032.safetensors", "model.layers.5.self_attn.k_proj.weight": "model-00006-of-00032.safetensors", "model.layers.5.self_attn.v_proj.weight": "model-00006-of-00032.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00006-of-00032.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00006-of-00032.safetensors", "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00007-of-00032.safetensors", "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00007-of-00032.safetensors", "model.layers.5.block_sparse_moe.experts.2.w3.weight": "model-00007-of-00032.safetensors", "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00007-of-00032.safetensors", "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00007-of-00032.safetensors", "model.layers.5.block_sparse_moe.experts.2.w1.weight": "model-00007-of-00032.safetensors", "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00032.safetensors", "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00032.safetensors", "model.layers.5.block_sparse_moe.experts.2.w2.weight": "model-00007-of-00032.safetensors", "model.layers.6.input_layernorm.weight": "model-00007-of-00032.safetensors", "model.layers.6.self_attn.q_proj.weight": "model-00007-of-00032.safetensors", "model.layers.6.self_attn.k_proj.weight": "model-00007-of-00032.safetensors", "model.layers.6.self_attn.v_proj.weight": "model-00007-of-00032.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00007-of-00032.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00007-of-00032.safetensors", "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00008-of-00032.safetensors", "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00008-of-00032.safetensors", "model.layers.6.block_sparse_moe.experts.2.w3.weight": "model-00008-of-00032.safetensors", "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00008-of-00032.safetensors", "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00008-of-00032.safetensors", "model.layers.6.block_sparse_moe.experts.2.w1.weight": "model-00008-of-00032.safetensors", "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00032.safetensors", "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00032.safetensors", "model.layers.6.block_sparse_moe.experts.2.w2.weight": "model-00008-of-00032.safetensors", "model.layers.7.input_layernorm.weight": "model-00008-of-00032.safetensors", "model.layers.7.self_attn.q_proj.weight": "model-00008-of-00032.safetensors", "model.layers.7.self_attn.k_proj.weight": "model-00008-of-00032.safetensors", "model.layers.7.self_attn.v_proj.weight": "model-00008-of-00032.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00008-of-00032.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00008-of-00032.safetensors", "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00009-of-00032.safetensors", "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00009-of-00032.safetensors", "model.layers.7.block_sparse_moe.experts.2.w3.weight": "model-00009-of-00032.safetensors", "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00032.safetensors", "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00032.safetensors", "model.layers.7.block_sparse_moe.experts.2.w1.weight": "model-00009-of-00032.safetensors", "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00009-of-00032.safetensors", "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00009-of-00032.safetensors", "model.layers.7.block_sparse_moe.experts.2.w2.weight": "model-00009-of-00032.safetensors", "model.layers.8.input_layernorm.weight": "model-00009-of-00032.safetensors", "model.layers.8.self_attn.q_proj.weight": "model-00009-of-00032.safetensors", "model.layers.8.self_attn.k_proj.weight": "model-00009-of-00032.safetensors", "model.layers.8.self_attn.v_proj.weight": "model-00009-of-00032.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00009-of-00032.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00009-of-00032.safetensors", "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00010-of-00032.safetensors", "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00010-of-00032.safetensors", "model.layers.8.block_sparse_moe.experts.2.w3.weight": "model-00010-of-00032.safetensors", "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00032.safetensors", "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00032.safetensors", "model.layers.8.block_sparse_moe.experts.2.w1.weight": "model-00010-of-00032.safetensors", "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00010-of-00032.safetensors", "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00010-of-00032.safetensors", "model.layers.8.block_sparse_moe.experts.2.w2.weight": "model-00010-of-00032.safetensors", "model.layers.9.input_layernorm.weight": "model-00010-of-00032.safetensors", "model.layers.9.self_attn.q_proj.weight": "model-00010-of-00032.safetensors", "model.layers.9.self_attn.k_proj.weight": "model-00010-of-00032.safetensors", "model.layers.9.self_attn.v_proj.weight": "model-00010-of-00032.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00010-of-00032.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00010-of-00032.safetensors", "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00011-of-00032.safetensors", "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00011-of-00032.safetensors", "model.layers.9.block_sparse_moe.experts.2.w3.weight": "model-00011-of-00032.safetensors", "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00032.safetensors", "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00032.safetensors", "model.layers.9.block_sparse_moe.experts.2.w1.weight": "model-00011-of-00032.safetensors", "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00011-of-00032.safetensors", "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00011-of-00032.safetensors", "model.layers.9.block_sparse_moe.experts.2.w2.weight": "model-00011-of-00032.safetensors", "model.layers.10.input_layernorm.weight": "model-00011-of-00032.safetensors", "model.layers.10.self_attn.q_proj.weight": "model-00011-of-00032.safetensors", "model.layers.10.self_attn.k_proj.weight": "model-00011-of-00032.safetensors", "model.layers.10.self_attn.v_proj.weight": "model-00011-of-00032.safetensors", "model.layers.10.self_attn.o_proj.weight": "model-00011-of-00032.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00011-of-00032.safetensors", "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00012-of-00032.safetensors", "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00012-of-00032.safetensors", "model.layers.10.block_sparse_moe.experts.2.w3.weight": "model-00012-of-00032.safetensors", "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00032.safetensors", "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00032.safetensors", "model.layers.10.block_sparse_moe.experts.2.w1.weight": "model-00012-of-00032.safetensors", "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00012-of-00032.safetensors", "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00012-of-00032.safetensors", "model.layers.10.block_sparse_moe.experts.2.w2.weight": "model-00012-of-00032.safetensors", "model.layers.11.input_layernorm.weight": "model-00012-of-00032.safetensors", "model.layers.11.self_attn.q_proj.weight": "model-00012-of-00032.safetensors", "model.layers.11.self_attn.k_proj.weight": "model-00012-of-00032.safetensors", "model.layers.11.self_attn.v_proj.weight": "model-00012-of-00032.safetensors", "model.layers.11.self_attn.o_proj.weight": "model-00012-of-00032.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00012-of-00032.safetensors", "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00013-of-00032.safetensors", "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00013-of-00032.safetensors", "model.layers.11.block_sparse_moe.experts.2.w3.weight": "model-00013-of-00032.safetensors", "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00013-of-00032.safetensors", "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00013-of-00032.safetensors", "model.layers.11.block_sparse_moe.experts.2.w1.weight": "model-00013-of-00032.safetensors", "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00013-of-00032.safetensors", "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00013-of-00032.safetensors", "model.layers.11.block_sparse_moe.experts.2.w2.weight": "model-00013-of-00032.safetensors", "model.layers.12.input_layernorm.weight": "model-00013-of-00032.safetensors", "model.layers.12.self_attn.q_proj.weight": "model-00013-of-00032.safetensors", "model.layers.12.self_attn.k_proj.weight": "model-00013-of-00032.safetensors", "model.layers.12.self_attn.v_proj.weight": "model-00013-of-00032.safetensors", "model.layers.12.self_attn.o_proj.weight": "model-00013-of-00032.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00013-of-00032.safetensors", "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00014-of-00032.safetensors", "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00014-of-00032.safetensors", "model.layers.12.block_sparse_moe.experts.2.w3.weight": "model-00014-of-00032.safetensors", "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00014-of-00032.safetensors", "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00014-of-00032.safetensors", "model.layers.12.block_sparse_moe.experts.2.w1.weight": "model-00014-of-00032.safetensors", "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00014-of-00032.safetensors", "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00014-of-00032.safetensors", "model.layers.12.block_sparse_moe.experts.2.w2.weight": "model-00014-of-00032.safetensors", "model.layers.13.input_layernorm.weight": "model-00014-of-00032.safetensors", "model.layers.13.self_attn.q_proj.weight": "model-00014-of-00032.safetensors", "model.layers.13.self_attn.k_proj.weight": "model-00014-of-00032.safetensors", "model.layers.13.self_attn.v_proj.weight": "model-00014-of-00032.safetensors", "model.layers.13.self_attn.o_proj.weight": "model-00014-of-00032.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00014-of-00032.safetensors", "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00015-of-00032.safetensors", "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00015-of-00032.safetensors", "model.layers.13.block_sparse_moe.experts.2.w3.weight": "model-00015-of-00032.safetensors", "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00015-of-00032.safetensors", "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00015-of-00032.safetensors", "model.layers.13.block_sparse_moe.experts.2.w1.weight": "model-00015-of-00032.safetensors", "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00015-of-00032.safetensors", "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00015-of-00032.safetensors", "model.layers.13.block_sparse_moe.experts.2.w2.weight": "model-00015-of-00032.safetensors", "model.layers.14.input_layernorm.weight": "model-00015-of-00032.safetensors", "model.layers.14.self_attn.q_proj.weight": "model-00015-of-00032.safetensors", "model.layers.14.self_attn.k_proj.weight": "model-00015-of-00032.safetensors", "model.layers.14.self_attn.v_proj.weight": "model-00015-of-00032.safetensors", "model.layers.14.self_attn.o_proj.weight": "model-00015-of-00032.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00015-of-00032.safetensors", "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00016-of-00032.safetensors", "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00016-of-00032.safetensors", "model.layers.14.block_sparse_moe.experts.2.w3.weight": "model-00016-of-00032.safetensors", "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00016-of-00032.safetensors", "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00016-of-00032.safetensors", "model.layers.14.block_sparse_moe.experts.2.w1.weight": "model-00016-of-00032.safetensors", "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00016-of-00032.safetensors", "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00016-of-00032.safetensors", "model.layers.14.block_sparse_moe.experts.2.w2.weight": "model-00016-of-00032.safetensors", "model.layers.15.input_layernorm.weight": "model-00016-of-00032.safetensors", "model.layers.15.self_attn.q_proj.weight": "model-00016-of-00032.safetensors", "model.layers.15.self_attn.k_proj.weight": "model-00016-of-00032.safetensors", "model.layers.15.self_attn.v_proj.weight": "model-00016-of-00032.safetensors", "model.layers.15.self_attn.o_proj.weight": "model-00016-of-00032.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00016-of-00032.safetensors", "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00017-of-00032.safetensors", "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00017-of-00032.safetensors", "model.layers.15.block_sparse_moe.experts.2.w3.weight": "model-00017-of-00032.safetensors", "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00017-of-00032.safetensors", "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00017-of-00032.safetensors", "model.layers.15.block_sparse_moe.experts.2.w1.weight": "model-00017-of-00032.safetensors", "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00017-of-00032.safetensors", "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00017-of-00032.safetensors", "model.layers.15.block_sparse_moe.experts.2.w2.weight": "model-00017-of-00032.safetensors", "model.layers.16.input_layernorm.weight": "model-00017-of-00032.safetensors", "model.layers.16.self_attn.q_proj.weight": "model-00017-of-00032.safetensors", "model.layers.16.self_attn.k_proj.weight": "model-00017-of-00032.safetensors", "model.layers.16.self_attn.v_proj.weight": "model-00017-of-00032.safetensors", "model.layers.16.self_attn.o_proj.weight": "model-00017-of-00032.safetensors", "model.layers.16.post_attention_layernorm.weight": "model-00017-of-00032.safetensors", "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00018-of-00032.safetensors", "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00018-of-00032.safetensors", "model.layers.16.block_sparse_moe.experts.2.w3.weight": "model-00018-of-00032.safetensors", "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00018-of-00032.safetensors", "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00018-of-00032.safetensors", "model.layers.16.block_sparse_moe.experts.2.w1.weight": "model-00018-of-00032.safetensors", "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00018-of-00032.safetensors", "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00018-of-00032.safetensors", "model.layers.16.block_sparse_moe.experts.2.w2.weight": "model-00018-of-00032.safetensors", "model.layers.17.input_layernorm.weight": "model-00018-of-00032.safetensors", "model.layers.17.self_attn.q_proj.weight": "model-00018-of-00032.safetensors", "model.layers.17.self_attn.k_proj.weight": "model-00018-of-00032.safetensors", "model.layers.17.self_attn.v_proj.weight": "model-00018-of-00032.safetensors", "model.layers.17.self_attn.o_proj.weight": "model-00018-of-00032.safetensors", "model.layers.17.post_attention_layernorm.weight": "model-00018-of-00032.safetensors", "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00019-of-00032.safetensors", "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00019-of-00032.safetensors", "model.layers.17.block_sparse_moe.experts.2.w3.weight": "model-00019-of-00032.safetensors", "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00019-of-00032.safetensors", "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00019-of-00032.safetensors", "model.layers.17.block_sparse_moe.experts.2.w1.weight": "model-00019-of-00032.safetensors", "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00019-of-00032.safetensors", "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00019-of-00032.safetensors", "model.layers.17.block_sparse_moe.experts.2.w2.weight": "model-00019-of-00032.safetensors", "model.layers.18.input_layernorm.weight": "model-00019-of-00032.safetensors", "model.layers.18.self_attn.q_proj.weight": "model-00019-of-00032.safetensors", "model.layers.18.self_attn.k_proj.weight": "model-00019-of-00032.safetensors", "model.layers.18.self_attn.v_proj.weight": "model-00019-of-00032.safetensors", "model.layers.18.self_attn.o_proj.weight": "model-00019-of-00032.safetensors", "model.layers.18.post_attention_layernorm.weight": "model-00019-of-00032.safetensors", "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00020-of-00032.safetensors", "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00020-of-00032.safetensors", "model.layers.18.block_sparse_moe.experts.2.w3.weight": "model-00020-of-00032.safetensors", "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00020-of-00032.safetensors", "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00020-of-00032.safetensors", "model.layers.18.block_sparse_moe.experts.2.w1.weight": "model-00020-of-00032.safetensors", "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00020-of-00032.safetensors", "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00020-of-00032.safetensors", "model.layers.18.block_sparse_moe.experts.2.w2.weight": "model-00020-of-00032.safetensors", "model.layers.19.input_layernorm.weight": "model-00020-of-00032.safetensors", "model.layers.19.self_attn.q_proj.weight": "model-00020-of-00032.safetensors", "model.layers.19.self_attn.k_proj.weight": "model-00020-of-00032.safetensors", "model.layers.19.self_attn.v_proj.weight": "model-00020-of-00032.safetensors", "model.layers.19.self_attn.o_proj.weight": "model-00020-of-00032.safetensors", "model.layers.19.post_attention_layernorm.weight": "model-00020-of-00032.safetensors", "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00021-of-00032.safetensors", "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00021-of-00032.safetensors", "model.layers.19.block_sparse_moe.experts.2.w3.weight": "model-00021-of-00032.safetensors", "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00021-of-00032.safetensors", "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00021-of-00032.safetensors", "model.layers.19.block_sparse_moe.experts.2.w1.weight": "model-00021-of-00032.safetensors", "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00021-of-00032.safetensors", "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00021-of-00032.safetensors", "model.layers.19.block_sparse_moe.experts.2.w2.weight": "model-00021-of-00032.safetensors", "model.layers.20.input_layernorm.weight": "model-00021-of-00032.safetensors", "model.layers.20.self_attn.q_proj.weight": "model-00021-of-00032.safetensors", "model.layers.20.self_attn.k_proj.weight": "model-00021-of-00032.safetensors", "model.layers.20.self_attn.v_proj.weight": "model-00021-of-00032.safetensors", "model.layers.20.self_attn.o_proj.weight": "model-00021-of-00032.safetensors", "model.layers.20.post_attention_layernorm.weight": "model-00021-of-00032.safetensors", "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00022-of-00032.safetensors", "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00022-of-00032.safetensors", "model.layers.20.block_sparse_moe.experts.2.w3.weight": "model-00022-of-00032.safetensors", "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00022-of-00032.safetensors", "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00022-of-00032.safetensors", "model.layers.20.block_sparse_moe.experts.2.w1.weight": "model-00022-of-00032.safetensors", "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00022-of-00032.safetensors", "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00022-of-00032.safetensors", "model.layers.20.block_sparse_moe.experts.2.w2.weight": "model-00022-of-00032.safetensors", "model.layers.21.input_layernorm.weight": "model-00022-of-00032.safetensors", "model.layers.21.self_attn.q_proj.weight": "model-00022-of-00032.safetensors", "model.layers.21.self_attn.k_proj.weight": "model-00022-of-00032.safetensors", "model.layers.21.self_attn.v_proj.weight": "model-00022-of-00032.safetensors", "model.layers.21.self_attn.o_proj.weight": "model-00022-of-00032.safetensors", "model.layers.21.post_attention_layernorm.weight": "model-00022-of-00032.safetensors", "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00023-of-00032.safetensors", "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00023-of-00032.safetensors", "model.layers.21.block_sparse_moe.experts.2.w3.weight": "model-00023-of-00032.safetensors", "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00023-of-00032.safetensors", "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00023-of-00032.safetensors", "model.layers.21.block_sparse_moe.experts.2.w1.weight": "model-00023-of-00032.safetensors", "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00023-of-00032.safetensors", "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00023-of-00032.safetensors", "model.layers.21.block_sparse_moe.experts.2.w2.weight": "model-00023-of-00032.safetensors", "model.layers.22.input_layernorm.weight": "model-00023-of-00032.safetensors", "model.layers.22.self_attn.q_proj.weight": "model-00023-of-00032.safetensors", "model.layers.22.self_attn.k_proj.weight": "model-00023-of-00032.safetensors", "model.layers.22.self_attn.v_proj.weight": "model-00023-of-00032.safetensors", "model.layers.22.self_attn.o_proj.weight": "model-00023-of-00032.safetensors", "model.layers.22.post_attention_layernorm.weight": "model-00023-of-00032.safetensors", "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00024-of-00032.safetensors", "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00024-of-00032.safetensors", "model.layers.22.block_sparse_moe.experts.2.w3.weight": "model-00024-of-00032.safetensors", "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00024-of-00032.safetensors", "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00024-of-00032.safetensors", "model.layers.22.block_sparse_moe.experts.2.w1.weight": "model-00024-of-00032.safetensors", "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00024-of-00032.safetensors", "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00024-of-00032.safetensors", "model.layers.22.block_sparse_moe.experts.2.w2.weight": "model-00024-of-00032.safetensors", "model.layers.23.input_layernorm.weight": "model-00024-of-00032.safetensors", "model.layers.23.self_attn.q_proj.weight": "model-00024-of-00032.safetensors", "model.layers.23.self_attn.k_proj.weight": "model-00024-of-00032.safetensors", "model.layers.23.self_attn.v_proj.weight": "model-00024-of-00032.safetensors", "model.layers.23.self_attn.o_proj.weight": "model-00024-of-00032.safetensors", "model.layers.23.post_attention_layernorm.weight": "model-00024-of-00032.safetensors", "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00025-of-00032.safetensors", "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00025-of-00032.safetensors", "model.layers.23.block_sparse_moe.experts.2.w3.weight": "model-00025-of-00032.safetensors", "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00025-of-00032.safetensors", "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00025-of-00032.safetensors", "model.layers.23.block_sparse_moe.experts.2.w1.weight": "model-00025-of-00032.safetensors", "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00025-of-00032.safetensors", "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00025-of-00032.safetensors", "model.layers.23.block_sparse_moe.experts.2.w2.weight": "model-00025-of-00032.safetensors", "model.layers.24.input_layernorm.weight": "model-00025-of-00032.safetensors", "model.layers.24.self_attn.q_proj.weight": "model-00025-of-00032.safetensors", "model.layers.24.self_attn.k_proj.weight": "model-00025-of-00032.safetensors", "model.layers.24.self_attn.v_proj.weight": "model-00025-of-00032.safetensors", "model.layers.24.self_attn.o_proj.weight": "model-00025-of-00032.safetensors", "model.layers.24.post_attention_layernorm.weight": "model-00025-of-00032.safetensors", "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00026-of-00032.safetensors", "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00026-of-00032.safetensors", "model.layers.24.block_sparse_moe.experts.2.w3.weight": "model-00026-of-00032.safetensors", "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00026-of-00032.safetensors", "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00026-of-00032.safetensors", "model.layers.24.block_sparse_moe.experts.2.w1.weight": "model-00026-of-00032.safetensors", "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00026-of-00032.safetensors", "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00026-of-00032.safetensors", "model.layers.24.block_sparse_moe.experts.2.w2.weight": "model-00026-of-00032.safetensors", "model.layers.25.input_layernorm.weight": "model-00026-of-00032.safetensors", "model.layers.25.self_attn.q_proj.weight": "model-00026-of-00032.safetensors", "model.layers.25.self_attn.k_proj.weight": "model-00026-of-00032.safetensors", "model.layers.25.self_attn.v_proj.weight": "model-00026-of-00032.safetensors", "model.layers.25.self_attn.o_proj.weight": "model-00026-of-00032.safetensors", "model.layers.25.post_attention_layernorm.weight": "model-00026-of-00032.safetensors", "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00027-of-00032.safetensors", "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00027-of-00032.safetensors", "model.layers.25.block_sparse_moe.experts.2.w3.weight": "model-00027-of-00032.safetensors", "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00027-of-00032.safetensors", "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00027-of-00032.safetensors", "model.layers.25.block_sparse_moe.experts.2.w1.weight": "model-00027-of-00032.safetensors", "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00027-of-00032.safetensors", "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00027-of-00032.safetensors", "model.layers.25.block_sparse_moe.experts.2.w2.weight": "model-00027-of-00032.safetensors", "model.layers.26.input_layernorm.weight": "model-00027-of-00032.safetensors", "model.layers.26.self_attn.q_proj.weight": "model-00027-of-00032.safetensors", "model.layers.26.self_attn.k_proj.weight": "model-00027-of-00032.safetensors", "model.layers.26.self_attn.v_proj.weight": "model-00027-of-00032.safetensors", "model.layers.26.self_attn.o_proj.weight": "model-00027-of-00032.safetensors", "model.layers.26.post_attention_layernorm.weight": "model-00027-of-00032.safetensors", "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00028-of-00032.safetensors", "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00028-of-00032.safetensors", "model.layers.26.block_sparse_moe.experts.2.w3.weight": "model-00028-of-00032.safetensors", "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00028-of-00032.safetensors", "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00028-of-00032.safetensors", "model.layers.26.block_sparse_moe.experts.2.w1.weight": "model-00028-of-00032.safetensors", "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00028-of-00032.safetensors", "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00028-of-00032.safetensors", "model.layers.26.block_sparse_moe.experts.2.w2.weight": "model-00028-of-00032.safetensors", "model.layers.27.input_layernorm.weight": "model-00028-of-00032.safetensors", "model.layers.27.self_attn.q_proj.weight": "model-00028-of-00032.safetensors", "model.layers.27.self_attn.k_proj.weight": "model-00028-of-00032.safetensors", "model.layers.27.self_attn.v_proj.weight": "model-00028-of-00032.safetensors", "model.layers.27.self_attn.o_proj.weight": "model-00028-of-00032.safetensors", "model.layers.27.post_attention_layernorm.weight": "model-00028-of-00032.safetensors", "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00029-of-00032.safetensors", "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00029-of-00032.safetensors", "model.layers.27.block_sparse_moe.experts.2.w3.weight": "model-00029-of-00032.safetensors", "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00029-of-00032.safetensors", "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00029-of-00032.safetensors", "model.layers.27.block_sparse_moe.experts.2.w1.weight": "model-00029-of-00032.safetensors", "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00029-of-00032.safetensors", "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00029-of-00032.safetensors", "model.layers.27.block_sparse_moe.experts.2.w2.weight": "model-00029-of-00032.safetensors", "model.layers.28.input_layernorm.weight": "model-00029-of-00032.safetensors", "model.layers.28.self_attn.q_proj.weight": "model-00029-of-00032.safetensors", "model.layers.28.self_attn.k_proj.weight": "model-00029-of-00032.safetensors", "model.layers.28.self_attn.v_proj.weight": "model-00029-of-00032.safetensors", "model.layers.28.self_attn.o_proj.weight": "model-00029-of-00032.safetensors", "model.layers.28.post_attention_layernorm.weight": "model-00029-of-00032.safetensors", "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00030-of-00032.safetensors", "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00030-of-00032.safetensors", "model.layers.28.block_sparse_moe.experts.2.w3.weight": "model-00030-of-00032.safetensors", "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00030-of-00032.safetensors", "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00030-of-00032.safetensors", "model.layers.28.block_sparse_moe.experts.2.w1.weight": "model-00030-of-00032.safetensors", "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00030-of-00032.safetensors", "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00030-of-00032.safetensors", "model.layers.28.block_sparse_moe.experts.2.w2.weight": "model-00030-of-00032.safetensors", "model.layers.29.input_layernorm.weight": "model-00030-of-00032.safetensors", "model.layers.29.self_attn.q_proj.weight": "model-00030-of-00032.safetensors", "model.layers.29.self_attn.k_proj.weight": "model-00030-of-00032.safetensors", "model.layers.29.self_attn.v_proj.weight": "model-00030-of-00032.safetensors", "model.layers.29.self_attn.o_proj.weight": "model-00030-of-00032.safetensors", "model.layers.29.post_attention_layernorm.weight": "model-00030-of-00032.safetensors", "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00031-of-00032.safetensors", "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00031-of-00032.safetensors", "model.layers.29.block_sparse_moe.experts.2.w3.weight": "model-00031-of-00032.safetensors", "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00031-of-00032.safetensors", "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00031-of-00032.safetensors", "model.layers.29.block_sparse_moe.experts.2.w1.weight": "model-00031-of-00032.safetensors", "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00031-of-00032.safetensors", "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00031-of-00032.safetensors", "model.layers.29.block_sparse_moe.experts.2.w2.weight": "model-00031-of-00032.safetensors", "model.norm.weight": "model-00031-of-00032.safetensors", "lm_head.weight": "model-00032-of-00032.safetensors", "model.layers.0.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.1.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.2.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.3.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.4.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.5.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.6.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.7.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.8.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.9.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.10.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.11.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.12.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.13.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.14.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.15.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.16.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.17.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.18.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.19.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.20.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.21.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.22.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.23.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.24.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.25.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.26.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.27.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.28.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors", "model.layers.29.block_sparse_moe.gate.weight": "model-00032-of-00032.safetensors"}}

special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<｜begin▁of▁sentence｜>"
+}

tokenizer_config.json

@@ -0,0 +1,137 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "100000": {
+      "content": "<｜begin▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100001": {
+      "content": "<｜end▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100002": {
+      "content": "ø",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100003": {
+      "content": "ö",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100004": {
+      "content": "ú",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100005": {
+      "content": "ÿ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100006": {
+      "content": "õ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100007": {
+      "content": "÷",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100008": {
+      "content": "û",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100009": {
+      "content": "ý",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100010": {
+      "content": "À",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100011": {
+      "content": "ù",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100012": {
+      "content": "Á",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100013": {
+      "content": "þ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100014": {
+      "content": "ü",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "legacy": true,
+  "model_max_length": 4096,
+  "pad_token": "<｜begin▁of▁sentence｜>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": null,
+  "use_default_system_prompt": false
+}