Upload folder using huggingface_hub

README.md

@@ -1,10 +1,10 @@
 ---
 base_model:
+- meta-llama/Llama-3.2-3B-Instruct
 - bunnycore/Llama-3.2-3B-ProdigyPlusPlus
+- meta-llama/Llama-3.2-3B
 - chuanli11/Llama-3.2-3B-Instruct-uncensored
-- meta-llama/Llama-3.2-3B-Instruct
 - huihui-ai/Llama-3.2-3B-Instruct-abliterated
-- meta-llama/Llama-3.2-3B
 library_name: transformers
 tags:
 - mergekit
@@ -18,15 +18,15 @@ This is a merge of pre-trained language models created using [mergekit](https://
 ## Merge Details
 ### Merge Method
 
-This model was merged using the [TIES](https://arxiv.org/abs/2306.01708) merge method using [bunnycore/Llama-3.2-3B-ProdigyPlusPlus](https://huggingface.co/bunnycore/Llama-3.2-3B-ProdigyPlusPlus) as a base.
+This model was merged using the [DARE](https://arxiv.org/abs/2311.03099) [TIES](https://arxiv.org/abs/2306.01708) merge method using [huihui-ai/Llama-3.2-3B-Instruct-abliterated](https://huggingface.co/huihui-ai/Llama-3.2-3B-Instruct-abliterated) as a base.
 
 ### Models Merged
 
 The following models were included in the merge:
-* [chuanli11/Llama-3.2-3B-Instruct-uncensored](https://huggingface.co/chuanli11/Llama-3.2-3B-Instruct-uncensored)
 * [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct)
-* [huihui-ai/Llama-3.2-3B-Instruct-abliterated](https://huggingface.co/huihui-ai/Llama-3.2-3B-Instruct-abliterated)
+* [bunnycore/Llama-3.2-3B-ProdigyPlusPlus](https://huggingface.co/bunnycore/Llama-3.2-3B-ProdigyPlusPlus)
 * [meta-llama/Llama-3.2-3B](https://huggingface.co/meta-llama/Llama-3.2-3B)
+* [chuanli11/Llama-3.2-3B-Instruct-uncensored](https://huggingface.co/chuanli11/Llama-3.2-3B-Instruct-uncensored)
 
 ### Configuration
 
@@ -34,18 +34,14 @@ The following YAML configuration was used to produce this model:
 
 ```yaml
 base_model:
-  model: bunnycore/Llama-3.2-3B-ProdigyPlusPlus
-merge_method: ties
+  model: huihui-ai/Llama-3.2-3B-Instruct-abliterated
+layer_range:
+- 0
+- 28
+merge_method: dare_ties
 merge_method_sequence:
 - dare_ties
-- slerp
 - ties
-models:
-- model: meta-llama/Llama-3.2-3B-Instruct
-- model: meta-llama/Llama-3.2-3B
-- model: chuanli11/Llama-3.2-3B-Instruct-uncensored
-- model: huihui-ai/Llama-3.2-3B-Instruct-abliterated
-- model: bunnycore/Llama-3.2-3B-ProdigyPlusPlus
 parameters:
   batch_size: 32
   density: 0.5
@@ -72,22 +68,38 @@ parameters:
     - 0
   - value: 0.5
   weight: 0.5
-slice:
-- models:
-  - meta-llama/Llama-3.2-3B-Instruct
-  - bunnycore/Llama-3.2-3B-ProdigyPlusPlus
-- models:
-  - meta-llama/Llama-3.2-3B-Instruct
-  - meta-llama/Llama-3.2-3B
-- models:
-  - chuanli11/Llama-3.2-3B-Instruct-uncensored
-  - huihui-ai/Llama-3.2-3B-Instruct-abliterated
-sources:
-- model: meta-llama/Llama-3.2-3B-Instruct
-- model: meta-llama/Llama-3.2-3B
-- model: chuanli11/Llama-3.2-3B-Instruct-uncensored
-- model: huihui-ai/Llama-3.2-3B-Instruct-abliterated
-- model: bunnycore/Llama-3.2-3B-ProdigyPlusPlus
+slices:
+- sources:
+  - density: 0.5
+    layer_range:
+    - 0
+    - 28
+    model: meta-llama/Llama-3.2-3B-Instruct
+    weight: 0.5
+  - density: 0.5
+    layer_range:
+    - 0
+    - 28
+    model: meta-llama/Llama-3.2-3B
+    weight: 0.5
+  - density: 0.5
+    layer_range:
+    - 0
+    - 28
+    model: chuanli11/Llama-3.2-3B-Instruct-uncensored
+    weight: 0.5
+  - density: 0.5
+    layer_range:
+    - 0
+    - 28
+    model: huihui-ai/Llama-3.2-3B-Instruct-abliterated
+    weight: 0.5
+  - density: 0.5
+    layer_range:
+    - 0
+    - 28
+    model: bunnycore/Llama-3.2-3B-ProdigyPlusPlus
+    weight: 0.5
 tokenizer_source: union
 
 ```

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "bunnycore/Llama-3.2-3B-ProdigyPlusPlus",
+  "_name_or_path": "huihui-ai/Llama-3.2-3B-Instruct-abliterated",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -33,7 +33,7 @@
   },
   "rope_theta": 500000.0,
   "tie_word_embeddings": true,
-  "torch_dtype": "float16",
+  "torch_dtype": "bfloat16",
   "transformers_version": "4.45.1",
   "use_cache": true,
   "vocab_size": 128256

mergekit_config.yml

@@ -1,16 +1,12 @@
 base_model:
-  model: bunnycore/Llama-3.2-3B-ProdigyPlusPlus
-merge_method: ties
+  model: huihui-ai/Llama-3.2-3B-Instruct-abliterated
+layer_range:
+- 0
+- 28
+merge_method: dare_ties
 merge_method_sequence:
 - dare_ties
-- slerp
 - ties
-models:
-- model: meta-llama/Llama-3.2-3B-Instruct
-- model: meta-llama/Llama-3.2-3B
-- model: chuanli11/Llama-3.2-3B-Instruct-uncensored
-- model: huihui-ai/Llama-3.2-3B-Instruct-abliterated
-- model: bunnycore/Llama-3.2-3B-ProdigyPlusPlus
 parameters:
   batch_size: 32
   density: 0.5
@@ -37,20 +33,36 @@ parameters:
     - 0
   - value: 0.5
   weight: 0.5
-slice:
-- models:
-  - meta-llama/Llama-3.2-3B-Instruct
-  - bunnycore/Llama-3.2-3B-ProdigyPlusPlus
-- models:
-  - meta-llama/Llama-3.2-3B-Instruct
-  - meta-llama/Llama-3.2-3B
-- models:
-  - chuanli11/Llama-3.2-3B-Instruct-uncensored
-  - huihui-ai/Llama-3.2-3B-Instruct-abliterated
-sources:
-- model: meta-llama/Llama-3.2-3B-Instruct
-- model: meta-llama/Llama-3.2-3B
-- model: chuanli11/Llama-3.2-3B-Instruct-uncensored
-- model: huihui-ai/Llama-3.2-3B-Instruct-abliterated
-- model: bunnycore/Llama-3.2-3B-ProdigyPlusPlus
+slices:
+- sources:
+  - density: 0.5
+    layer_range:
+    - 0
+    - 28
+    model: meta-llama/Llama-3.2-3B-Instruct
+    weight: 0.5
+  - density: 0.5
+    layer_range:
+    - 0
+    - 28
+    model: meta-llama/Llama-3.2-3B
+    weight: 0.5
+  - density: 0.5
+    layer_range:
+    - 0
+    - 28
+    model: chuanli11/Llama-3.2-3B-Instruct-uncensored
+    weight: 0.5
+  - density: 0.5
+    layer_range:
+    - 0
+    - 28
+    model: huihui-ai/Llama-3.2-3B-Instruct-abliterated
+    weight: 0.5
+  - density: 0.5
+    layer_range:
+    - 0
+    - 28
+    model: bunnycore/Llama-3.2-3B-ProdigyPlusPlus
+    weight: 0.5
 tokenizer_source: union

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a912a44fc0022a36e49beb59c5a6f2d3db36dcb5803079caab04900f2854c167
-size 4998794808
+oid sha256:890b30c2fcdc8d81e207bcad23b051bb66cb68123b37b057e6f065223669b88a
+size 4998794968

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:186db3644ffe2e65b859971d8f6b06677e7078da924701171e8ba81a486fe336
-size 2214738976
+oid sha256:bd563d25a47422404bd7102fb0c065c572032403559e7ce84fa64c0654469ed4
+size 2214739072