Hugging Face's logo

Sakalti
/
SJT-Moe2x7.5B

Text Generation
Transformers
Safetensors
mixtral
Mixture of Experts
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
Sakalti commited on
Commit
419338a
·
verified ·
1 Parent(s): bbe34ad

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +37 -0
  2. mergekit_moe_config.yml +8 -0
  3. model-00001-of-00059.safetensors +3 -0
  4. model-00002-of-00059.safetensors +3 -0
  5. model-00003-of-00059.safetensors +3 -0
  6. model-00004-of-00059.safetensors +3 -0
  7. model-00005-of-00059.safetensors +3 -0
  8. model-00006-of-00059.safetensors +3 -0
  9. model-00007-of-00059.safetensors +3 -0
  10. model-00008-of-00059.safetensors +3 -0
  11. model-00009-of-00059.safetensors +3 -0
  12. model-00010-of-00059.safetensors +3 -0
  13. model-00011-of-00059.safetensors +3 -0
  14. model-00012-of-00059.safetensors +3 -0
  15. model-00013-of-00059.safetensors +3 -0
  16. model-00014-of-00059.safetensors +3 -0
  17. model-00015-of-00059.safetensors +3 -0
  18. model-00016-of-00059.safetensors +3 -0
  19. model-00017-of-00059.safetensors +3 -0
  20. model-00018-of-00059.safetensors +3 -0
  21. model-00019-of-00059.safetensors +3 -0
  22. model-00020-of-00059.safetensors +3 -0
  23. model-00021-of-00059.safetensors +3 -0
  24. model-00022-of-00059.safetensors +3 -0
  25. model-00023-of-00059.safetensors +3 -0
  26. model-00024-of-00059.safetensors +3 -0
  27. model-00025-of-00059.safetensors +3 -0
  28. model-00026-of-00059.safetensors +3 -0
  29. model-00027-of-00059.safetensors +3 -0
  30. model-00028-of-00059.safetensors +3 -0
  31. model-00029-of-00059.safetensors +3 -0
  32. model-00030-of-00059.safetensors +3 -0
  33. model-00031-of-00059.safetensors +3 -0
  34. model-00032-of-00059.safetensors +3 -0
  35. model-00033-of-00059.safetensors +3 -0
  36. model-00034-of-00059.safetensors +3 -0
  37. model-00035-of-00059.safetensors +3 -0
  38. model-00036-of-00059.safetensors +3 -0
  39. model-00037-of-00059.safetensors +3 -0
  40. model-00038-of-00059.safetensors +3 -0
  41. model-00039-of-00059.safetensors +3 -0
  42. model-00040-of-00059.safetensors +3 -0
  43. model-00041-of-00059.safetensors +3 -0
  44. model-00042-of-00059.safetensors +3 -0
  45. model-00043-of-00059.safetensors +3 -0
  46. model-00044-of-00059.safetensors +3 -0
  47. model-00045-of-00059.safetensors +3 -0
  48. model-00046-of-00059.safetensors +3 -0
  49. model-00047-of-00059.safetensors +3 -0
  50. model-00048-of-00059.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Sakalti/SJT-7.5B",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 11,
9
+ "eos_token_id": 11,
10
+ "head_dim": 256,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 3072,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 23040,
15
+ "max_position_embeddings": 32768,
16
+ "mlp_bias": false,
17
+ "model_type": "mixtral",
18
+ "num_attention_heads": 12,
19
+ "num_experts_per_tok": 2,
20
+ "num_hidden_layers": 28,
21
+ "num_key_value_heads": 4,
22
+ "num_local_experts": 2,
23
+ "output_router_logits": false,
24
+ "pad_token_id": 2023,
25
+ "pretraining_tp": 1,
26
+ "rms_norm_eps": 1e-06,
27
+ "rope_scaling": null,
28
+ "rope_theta": 1000042,
29
+ "router_aux_loss_coef": 0.001,
30
+ "router_jitter_noise": 0.0,
31
+ "sliding_window": null,
32
+ "tie_word_embeddings": false,
33
+ "torch_dtype": "float16",
34
+ "transformers_version": "4.46.3",
35
+ "use_cache": true,
36
+ "vocab_size": 131072
37
+ }
mergekit_moe_config.yml ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ base_model: Sakalti/SJT-7.5B
2
+ gate_mode: random
3
+ dtype: float16
4
+ experts:
5
+ - source_model: Sakalti/SJT-7.5B
6
+ positive_prompts: [hello, こんにちは, おはようございます。,お元気ですか？]
7
+ - source_model: Sakalti/SJT-7.5B
8
+ negative_prompts: [bad]
model-00001-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:64361f97fdddceb0c78323c43e067299a9bc2142fe8f705141b27ed7711d430d
3
+ size 805306504
model-00002-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b35801653e374d658d5bbce8b6c7d9796f0912b138bec8c29656af17d8c33ac
3
+ size 475018320
model-00003-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e93a1bb46ad2e2b9867b49d21ccee1fda8a9ee7f8302c437ceb2f25300b904a9
3
+ size 475018320
model-00004-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c407bb98cdb2d3e8e0fdd04c0ac9a995ca1e521c413ada70c582e6f58b11a34a
3
+ size 424673696
model-00005-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03d196df0f328772bf87532d4249265b0af1270206e9933aa32bede2551ab2cb
3
+ size 475018320
model-00006-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5cb6d4a33696e1af408ae7990fe6ff084fc419b04e66ed426d3353134b52b692
3
+ size 424673696
model-00007-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:865eb9a171a26f724f46a8e38ecf4ecf256f2d3fef5b92db06bb354f32f215fd
3
+ size 475018320
model-00008-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9c0e3c9727c778d3e400322947e2e5a44b1eeb781e620a460eaebbff2047f9d
3
+ size 424673696
model-00009-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c2ee41d52d271527067635e8e9d4b2fb4878119f80d058ccbc5bf35f1f44747
3
+ size 475018320
model-00010-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36f32607b6a33b969389021c24d0bad012d65b81a89bcbee6051be1d863e57bd
3
+ size 424673696
model-00011-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:770437b70191cc18fd88a287ad21301c769751be7645a7ce6ff26da68771f972
3
+ size 475018320
model-00012-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7cb9e9848c61efbb55b5e448b04861bc5cfaf62efec621988e5227c559e366b4
3
+ size 424673696
model-00013-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13fd36302781b134e544f25c301dd820537f0fcd16778dec99cdaa82dcf5c558
3
+ size 475018320
model-00014-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c918c0359c83c3f52e01f7d4ecf95cee880597be7642db7f686af6d9f96662c8
3
+ size 424673696
model-00015-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:704efca01c30d753ab4efc3cac9b8437047cc674b743d6496ddda0f24e5750f1
3
+ size 475018320
model-00016-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1404dcd781837dc05d961e0f0ee9c5239650f850b865ae3c723e075831507514
3
+ size 424673696
model-00017-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7672002fd91851c403170ae73015996b8e6f5fb532623c490e8f2182f05c58d7
3
+ size 475018320
model-00018-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d371d363267076edd817f92066bf7c560350b60d80b83a3bc34e120b6098954f
3
+ size 424673696
model-00019-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9454821f32aee2e13fd5ca75f1024943a14b61e410704fe413424393438b2312
3
+ size 475018320
model-00020-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:120439e4e05107aaed6bd14d20ad7b380aef8663c56ce61a6b4fa7731512080e
3
+ size 424673696
model-00021-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6dc05b916134f7b8ca553c50eb0a9a29eaa7add53add41ed692d854580f12c29
3
+ size 475018296
model-00022-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ddbd9501a77431ae1c9a764d55789b352e5dc2f9f260d83da44dc0905c17fa3d
3
+ size 424673704
model-00023-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8001b75823d9594c889a16af66cd04159318493e9eb40f5b9e3ed103c184456
3
+ size 475018328
model-00024-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e618b5207a23e97f2dcbff9733a6add8153a9b21cf8a52e13bc96e5c4acfc8d5
3
+ size 424673704
model-00025-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3f9c2bfa1750d434343b92b4c9c8b4d25a1f16287d9e90c1965b46f3fa77b1e
3
+ size 475018328
model-00026-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdd940bfc755eccc82ec809a0fd4388ad0301c0f89afb01c3fd1acdbf5a6a132
3
+ size 424673704
model-00027-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd185da1df913f556de3a4c42022186a48c71dc1814b171d48a14d1b77c0acba
3
+ size 475018328
model-00028-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9e5ff6a0a17913b25b609a5bcd076e037559863deb15b8dfe980a5e72a10862
3
+ size 424673704
model-00029-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f12c92b41d7a51003696ca65798644df1157965ab1b2fee0d4cfc462ae03441c
3
+ size 475018328
model-00030-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:abf74ad8efaa982be5fda7adb73148db2d8beab4b982e9da9ccb9192d1880a00
3
+ size 424673704
model-00031-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8d41a1c89076bad44c8266b0b25db5846b33870e85d39337332b6623fdb6c8e
3
+ size 475018328
model-00032-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac4ad93b797464e2b1df2a5a83df3fef1ec7b0c7a1af7802e7e80ca84a300ee3
3
+ size 424673704
model-00033-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01086b2d2f090e9bbbdf4f854fcfd1b2520af42b33c8378b02d5ed8346db526b
3
+ size 475018328
model-00034-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e371907b1cd5f320779477d722e5cce42ab8656cf03407639d4f859bd13e81e2
3
+ size 424673704
model-00035-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:691aabfccfb94c0b3f72589bda6f5ec816df17e0d67e949f0e990603d2eee5c0
3
+ size 475018328
model-00036-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b33dad7bb8ac731daffa0138fce89d76cabaa204e6679824c5f7bbfae11d2ab
3
+ size 424673704
model-00037-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b6216d7ff527fa45a52e154898445f8cdf4a8b64f3b645dacbed28df9126fa2
3
+ size 475018328
model-00038-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec01a7b1c75ceb87803908912de58f44584d01c8b10b344dd4bdde26551ce9d4
3
+ size 424673704
model-00039-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f8f7fe5b3d16e9f798ec6d05147db4edb5117c14e5226cb1c594fab2f2bf8de
3
+ size 475018328
model-00040-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b4e9e65078da9aa3083d211ad23dcf22a55fd8d2cddd679737f19fde1b031f5
3
+ size 424673704
model-00041-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b6ad887d389559bf0e031b479d95e43b4422ddf6c8dde0b90f92d402c4a6d58
3
+ size 475018328
model-00042-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04590ce0a158cec894a3d9363996b4ee3a110dc99bbd910033314fa48133cbdd
3
+ size 424673704
model-00043-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9928f18c696490f13f1d6ad9f94219b8ccfc048a91535e22b65fad85dd484f3
3
+ size 475018328
model-00044-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de5771cec4bb3114b199e6b88e08996f042610efba0640c953d5fa41aec90954
3
+ size 424673704
model-00045-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c6c39ff689761cb8082052284709ccafef2437d5534d1106a1d7903d8561f81
3
+ size 475018328
model-00046-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06b196a749d885994ddcd46aa8c03dfc299f46145140157634e7a308f15ae03c
3
+ size 424673704
model-00047-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f192b10525fbd2f9144a984bdf6f21af3a444664f7609312b7d06ac3bf5eb302
3
+ size 475018328
model-00048-of-00059.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a749082482976f3289d848b7e09ebfafe71e1be66a7834fe6b53a83c2d1fe1f3
3
+ size 424673704