Upload folder using huggingface_hub

README.md

@@ -7,19 +7,15 @@ tags:
 - lazymergekit
 - cognitivecomputations/dolphin-2_6-phi-2
 - lxuechen/phi-2-dpo
-- Yhyu13/phi-2-sft-dpo-gpt4_en-ep1
-- mrm8488/phi-2-coder
 ---
 
 ![](https://i.imgur.com/UOb2fvh.jpg)
 
-# phixtral-4x2.8
+# phixtral-2x2.8
 
 phixtral-2x2.8 is a Mixure of Experts (MoE) made with the following models using a custom version of mergekit:
 * [cognitivecomputations/dolphin-2_6-phi-2](https://huggingface.co/cognitivecomputations/dolphin-2_6-phi-2)
 * [lxuechen/phi-2-dpo](https://huggingface.co/lxuechen/phi-2-dpo)
-* [Yhyu13/phi-2-sft-dpo-gpt4_en-ep1](https://huggingface.co/Yhyu13/phi-2-sft-dpo-gpt4_en-ep1)
-* [mrm8488/phi-2-coder](https://huggingface.co/mrm8488/phi-2-coder)
 
 ## 🧩 Configuration
 
@@ -31,10 +27,6 @@ experts:
     positive_prompts: [""]
   - source_model: lxuechen/phi-2-dpo
     positive_prompts: [""]
-  - source_model: Yhyu13/phi-2-sft-dpo-gpt4_en-ep1
-    positive_prompts: [""]
-  - source_model: mrm8488/phi-2-coder
-    positive_prompts: [""]
 ```
 
 ## 💻 Usage

config.json

@@ -34,7 +34,7 @@
   "num_experts_per_tok": 2,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
-  "num_local_experts": 4,
+  "num_local_experts": 2,
   "output_router_logits": false,
   "resid_pdrop": 0.1,
   "rms_norm_eps": 1e-06,

mergekit_moe_config.yml

@@ -4,8 +4,4 @@ experts:
   - source_model: cognitivecomputations/dolphin-2_6-phi-2
     positive_prompts: [""]
   - source_model: lxuechen/phi-2-dpo
-    positive_prompts: [""]
-  - source_model: Yhyu13/phi-2-sft-dpo-gpt4_en-ep1
-    positive_prompts: [""]
-  - source_model: mrm8488/phi-2-coder
     positive_prompts: [""]

model-00001-of-00001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca5d85304bec3b0f81ea5c166c72f14208ff1640e02c8cdf9f23572ee949a821
+size 8916015136