Upload folder using huggingface_hub

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "BioMistral/BioMistral-7B",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 6,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

mergekit_moe_config.yml

@@ -0,0 +1,24 @@
+base_model: BioMistral/BioMistral-7B
+gate_mode: approach # one of "hidden", "cheap_embed", or "random"
+dtype: bfloat16 # output dtype (float32, float16, or bfloat16)
+## (optional)
+# experts_per_token: 2
+experts:
+  - source_model: Treza12/BioMistral-Unsloth-Full-Class-0
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert model excels at Screening"
+  - source_model: Treza12/BioMistral-Unsloth-Full-Class-1
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert model excels at Diagnosis"
+  - source_model: Treza12/BioMistral-Unsloth-Full-Class-2
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert model excels at Staging"
+  - source_model: Treza12/BioMistral-Unsloth-Full-Class-33
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert model excels at Treatment Selection"
+  - source_model: Treza12/BioMistral-Unsloth-Full-Class-44
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert model excels at Prognosis" 
+  - source_model: Treza12/BioMistral-Unsloth-Full-Class-55
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert model excels at Follow-up" 

model-00001-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91ead4371615d6120af855b56d5fbc085864f702074411a15664c4c42e82af33
+size 933250496

model-00002-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0d4d7cd024ac318815693c08887ca48253876f45575869772ca52b3b44247eb
+size 939525160

model-00003-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65e5a2cac3ae6408e514143aed8b9979a9a860d7365dbb84e3f6e6e38c0023bd
+size 905987672

model-00004-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66f3a2f1242b25eeeb769ca813dfc1931445536dd0e0981529bb6e72607f44e3
+size 939525160

model-00005-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a89c6c10cc21d2e21488eb6b601a650ec0627fbfcc1b9dc203f01f5b5f5d4bdb
+size 989865456

model-00006-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e92bfc592bcede6a67caf1059f375d812a189f29c89e63f3db6065de9403359
+size 973088024

model-00007-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89d50c8f664ac664ce6ddc7b70cbfcb52060f9a9c2d3838208beaadf78ba1610
+size 939525160

model-00008-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29160e7dea4917b11299a327145b8a43082dfd4cb1801fa27c8c2c1af1ea0791
+size 905987672

model-00009-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81c8003fd7dd8ab66ecab91ac6d39039c9c211f823c8647872b5b18c93cb9c65
+size 939525160

model-00010-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccc8d88f913ddc06abfd8c1cd7d146daec41b4f9e2b373859b024b0c95c536d4
+size 905987672

model-00011-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf6b3c99a0e209d9621bcfc6ad7199c96103150a10cb068f12ece4cdc9c3fc59
+size 939525160

model-00012-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11cad6bc9a179993d6f1d9ac5e16f663aa6595ab38a86744706aa20cc7c56b78
+size 989865456

model-00013-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e9f3724b48f6a65357afdb3c62c7df7b974dc59b266163683e8381d9b6ecb46
+size 973088024

model-00014-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57ac0b575b1a73651ddd97b1619d33fd21c81378a64ebea2bd05fe4829ba31d7
+size 939525160

model-00015-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0acdd8e8fc838d30d098f6a75ceaf61a8ff1054a73d2e6043c62534507cafe9f
+size 905987672

model-00016-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a685716f55fdc0aba72726e5167ee48754f85a30bf21c9f6d90cd729651e3a2
+size 939525160

model-00017-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c1d42ff2a85aeec129480eb46c058f6db0f0b9bab89a34a0febbf5c8ff0525e
+size 905987672

model-00018-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85fd961d540f452fe711611b2a220d79f0f01ba18782ae58e752aae6cddd5ef3
+size 939525160

model-00019-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72d9d79f352719850f6b8167a9450c5537a5840194d4b19914b753129ebb4a29
+size 989865456

model-00020-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08072cd2d481473e36438c1bf906089bcca9c0f8fc177cc67ab95e259309678e
+size 973088024

model-00021-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da7ddb64133c485f98dad821a2b52d1f3a2d4dead62f0045e45badf95b7a1097
+size 939525160

model-00022-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71abb29f14ca3249fe14c0fde0334c8ebbae1d47d79a592ebdcdbda1da7e4231
+size 905987672

model-00023-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d02454e082e19ee8ffb7f30cdd8a5fa273f983d90a5bec4b5fa5b7c403421425
+size 939525160

model-00024-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b70275bfbfab64851d8b56121e79824cfa9922758cc2854d6a9d0d2b7201074d
+size 905987680

model-00025-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7ce71d246e86ce7d7fdef0f321d72dfd4cbd036fc1a1ad7da5ac924845e81fe
+size 939525168

model-00026-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98b3f3b5655f89c16dee1c4ad3be2f6027d8a250541e3335a8979f90e644f080
+size 989865472

model-00027-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfee08194746c7ab2f2007213a262e266867d5cfc0348f3e909b3c4c6a186127
+size 973088032

model-00028-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fa81df29c115db28f06c5c5ddd129b01226f1f448137b51f4bcf734d841217e
+size 939525168

model-00029-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b3988cff065d937d71d692fd4ee5712fc9ec6cab7325980f6d112aa0453dc16
+size 905987688

model-00030-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3bd27b8f737fa87e8e72aaab7c86a9b5e0ef8d1eacee27aa4d2edfd358fbce1
+size 939525168

model-00031-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c88588dcbca86c53e1f83769dc56a4a1595fb90e4debf9c19db834ff96f173cf
+size 905987688

model-00032-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bbd5990a0323dea879332fb0892b1a196c765d0c8c3528527cb4bb0c2a5c54e
+size 939525168

model-00033-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3784183831745c071e899b91b574ba558e964a27ef4b1b217d9b639bfa9a5a85
+size 989865472

model-00034-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75d0425ed2787f478b9f1f1c0fd229532630f436accad2256b1a9227200145ad
+size 973088032

model-00035-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76fd6aa7c5d98f43bcb4f4d3ad856e13507b44717b47fe4d5e639a9a3bb2964a
+size 939525168

model-00036-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a7e58a871e8a311dcb48be1c7790d421dc01663e056fe436cbb11d74bbccdd3
+size 905987688

model-00037-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04070baacc8c4ce3fea847ad2c33cab71d9bbe8d3d7a14e855152b6d5f8b6785
+size 939525168

model-00038-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1537f637b2d50e849b4ea1001d4d2d4a73fa3855cd051b2d2caac15cfe40102
+size 905987688

model-00039-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:008f0c1177748eb3d3dedbc5883b59eac8e657f8ae04aabee2bf4e151c9ba3d0
+size 939525168

model-00040-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6752493394d4806a3e6991d7b37aa8bd710c34589f2b41dde4fcc66d2c9642b3
+size 989865472

model-00041-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08d4c17eae6b6f1986329183f59b50005336ea95664e3b7611235d8b4710fbaf
+size 973088032

model-00042-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6e7c0a7be81a164e5f7eb6cc4fbf6deaef85bc5682f174cc9a7dfbcf334db36
+size 939525168

model-00043-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5623622857e6ccda9a5224c7731967db4451056941224d7a2b6219b6021b8e96
+size 905987688

model-00044-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da91e47bd130cda91a37219ff917d53a41ebb22c2e580e516b18ebc4b480b1e2
+size 939525168

model-00045-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84077a7940c09951e75d5a93712ec75bd7c488856270d2f33b83640b6b9c8cee
+size 905987688

model-00046-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4956c32302b0917115035225f3a364d9ba12236cea1f849ec0b44cd449f03af7
+size 939525168

model-00047-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4ae6ee0845d986578cb065688fe54eb44da5d901ada02945eef3133ce206562
+size 989865472

model-00048-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2148a81110bb7302e328541ee061c24fe77d0e32f7227883aa1c4803aa122366
+size 973088032