Upload folder using huggingface_hub

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "BioMistral/BioMistral-7B",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 6,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

mergekit_moe_config.yml

@@ -0,0 +1,22 @@
+base_model: BioMistral/BioMistral-7B
+gate_mode: approach # one of "hidden", "cheap_embed", or "random"
+dtype: bfloat16 # output dtype (float32, float16, or bfloat16)
+experts:
+  - source_model: Treza12/BioMistral-7B-F0
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert_model_1 excels at Screening"
+  - source_model: Treza12/BioMistral-7B-F1
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert_model_1 excels at Diagnosis"
+  - source_model: Treza12/BioMistral-7B-F2
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert_model_1 excels at Staging"
+  - source_model: Treza12/BioMistral-7B-F3
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert_model_1 excels at Treatment Selection"
+  - source_model: Treza12/BioMistral-7B-F4
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert_model_1 excels at Prognosis"
+  - source_model: Treza12/BioMistral-7B-F5
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert_model_1 excels at Follow-up"

model-00001-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7ff07b9f9e4cdbdcdfca15ca0b0aafee83e3dfb624d9d8cf6a5837fb62eaf8
+size 933250496

model-00002-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78747a20504f305157d14433d8e001d1616ccd5f0ac2cbb2a28344f64ff23344
+size 939525160

model-00003-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b5d20f7590223075697c8be5fc7a98f0e650886a4ea4065e12cfa6b2a2b8f27
+size 905987672

model-00004-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eb9d08aa17ebf4c466c7eabfcd609c748bc99a73f0f41670f909e1e985ea7fe
+size 939525160

model-00005-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f9893fe5b4cafa58891d4c76bdd92415804553e710daf2bdda14414f99dd2d4
+size 989865456

model-00006-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7851c9192f725f7527da283ed03dff509595e4fdc543bc0f325718b55ad6c5e7
+size 973088024

model-00007-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e259b243c2b3e703f9bbbdfe43b91fbfe202d9a5ee0b7973fc141fbffa06e8b
+size 939525160

model-00008-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:110cf51cc6dac6b70941b8c0a008be9fe740d7f1191746bd7611b772b31d2502
+size 905987672

model-00009-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b054f10876be379be023e2e687b78dfcb036b895f55ee87ff4da35ed7fb83ac
+size 939525160

model-00010-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dc97a38f3a4756d7ec32e5b71108a69f5e80d40b0fca40b5e2d645de13600df
+size 905987672

model-00011-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0845ba2a6390542e65cbb4c102c0dcdc43674e70264b566bcf67012e9035bff
+size 939525160

model-00012-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0756ee6d84a24755a8d8857743ef32c66acaeb1c48b2a359b13d7e8ea02d8896
+size 989865456

model-00013-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e49c815dfc2238daea2c83d1646c302955de2abdbc089fb0d8fed525b4175c6
+size 973088024

model-00014-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:591dd2eb620f9c01f25cb3e6427ae59ec0b68b80366085ed38c958ca8129e173
+size 939525160

model-00015-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bb808ea4a469dfdc2e1bd2fb37db72dfb2ca78af9ac8a79da8c55a04b3cc359
+size 905987672

model-00016-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4bed967ae5632a46ef069182cf59fc890da4297d6b7ce6305b87eb1bd278b15
+size 939525160

model-00017-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cc1ca6ddb4033c1a77d72e800f7db4757bf625ca4ffe63cecc3d7f1578689cb
+size 905987672

model-00018-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:defbe1d6aa13b6991b652b2d010bf23abcc4f883f769d642ab635e541583b358
+size 939525160

model-00019-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20f3f6a9936e013ff4f13ce0f38dc02dee616010c8d23996a71fb53f79d53843
+size 989865456

model-00020-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8827b73fc6c6fb693f58c827fa6bec817762916bc55d441663ae134b00245298
+size 973088024

model-00021-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51551d303cc09725404c0a31309c38edaadb6c848fc228bd4f82a2363d3bcc29
+size 939525160

model-00022-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cc6537860b0ae6e07d13e7d3f6527e43215b2db52afda519e8ad44df8c22802
+size 905987672

model-00023-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b3fb6f5042d258948c117dcb2279eeaef574438e67f6913f0b16ef786031c7d
+size 939525160

model-00024-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a2853917ad12d280de1aa09c22dbcd1063da400592f362908eb8e7d0fce82d0
+size 905987680

model-00025-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ece1f9c664da8491ed265b6648237b0ce7ecc0f1dcc8bf81e3ad8e61fc9dd1
+size 939525168

model-00026-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2345a1f9aa8de7336a181f92f9b68d6ab7d6d3d5429ed01b24582f416950a0b
+size 989865472

model-00027-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:671fdc9103a5dcec57ed6e8ea78e4acdd7ce8b2a3a9ef349568dac445b5fc72f
+size 973088032

model-00028-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21d5fd5606fdc95432c3d06a6d01b6feeb908eb75bd04ee8519046b6bd83d4e9
+size 939525168

model-00029-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee0e9a8e8581d5f61bea5dc4afe9285b198b03ee3b58d50fe3cadb460a7cf11f
+size 905987688

model-00030-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bfe03a947dea49c343e291f64a7fa5acaf74c493bd82592d6b4e4a4fd29612d
+size 939525168

model-00031-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cb59fcee85856e4b814eeb1373139f2893c52b3f5e380289125b528d7693078
+size 905987688

model-00032-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f59fdebccaea852a69ecbd3b48bd83f94f1e7baf46db49f9acc7c805ee0d1a52
+size 939525168

model-00033-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38a4fd19900f8bc79cd0b4aeac76a6887ba1ec09a17fc3dfdcf770a04bc493a3
+size 989865472

model-00034-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b06d0dc24f3921d965e721ffc329af4cbb307b8ab2fddf792c2daf1f4e08022
+size 973088032

model-00035-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3f0f5362566a4fc950ac91e459e2fbbf71806b5c76bf622426ccba2b8a09908
+size 939525168

model-00036-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8103dca2f2666b7c630f42a74fa2c0cb959d5b441d7d1af9b059531f84706c72
+size 905987688

model-00037-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e601c26b1f6e3217cb451f25da96be76e5677852e4468f53c1c9785e7cc30a37
+size 939525168

model-00038-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9de923954e0f73fc54e17839253c9e3a8103cff9f600cee268aa2de34f867087
+size 905987688

model-00039-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1e8dad1ea9aa8cf1f0fdbd407eb62d652a92ebdfdfe9097cad0b5c6469a8b1c
+size 939525168

model-00040-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a57cb654b9f3f0e71bd35166ef97e3c31bbd210dc3499a6a9f0efe301debd6fa
+size 989865472

model-00041-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ce38ad1eee9a8f5335538d27e839033817370a79e1fd2cccb3fe1d039c96783
+size 973088032

model-00042-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:174f42c05053c0d023dc723d5ee011d86e93dff2114a29a97171f39da6c7efde
+size 939525168

model-00043-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2109f23b25d91a41e88643f7f680fa5b3dfee1fcec90fbb85ff3b4029e358646
+size 905987688

model-00044-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb2537d86edda1c9a57e7858338829b96cbee78ce2cbdea1d3dc698dee8149f5
+size 939525168

model-00045-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58068f5a21c4dc516035453f2e7236544b9390078d7efab98e743cca1d87b190
+size 905987688

model-00046-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e803e85d74b57c5cf99d2b61791d9983e27296e4f1ce258a6309ac5b47e454df
+size 939525168

model-00047-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c11daff578aff817c4fbabf9df8dba0e7757ebc538a0df41c30accf6e20dca5f
+size 989865472

model-00048-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c903d255b6c416fed551703776b67721a98583deecb358d315ec44bc9a58a43
+size 973088032