| # Optimized MergeKit configuration for merging extracted LoRA adapters | |
| # into the Qwen2.5-14B-Instruct model. | |
| base_model: Qwen/Qwen2.5-14B | |
| models: | |
| # Each adapter was extracted (rank=128) from its respective finetuned model. | |
| # Their weights are set lower than the full instruct model (which is now the base) | |
| - model: CultriX/Qwen2.5-14B-Hyperionv3_r128 | |
| parameters: | |
| weight: 0.15 # Reduced weight relative to base | |
| density: 1.0 | |
| lora_rank: 128 # Mark as extracted LoRA adapter | |
| - model: CultriX/Qwen2.5-14B-Coder_r128 | |
| parameters: | |
| weight: 0.15 | |
| density: 1.0 | |
| lora_rank: 128 | |
| - model: CultriX/Qwen2.5-14B_Virtuoso-small-v2-LoRA_r128 | |
| parameters: | |
| weight: 0.15 | |
| density: 1.0 | |
| lora_rank: 128 | |
| - model: CultriX/Qwen2.5-14B-SuperNova-Medius_r128 | |
| parameters: | |
| weight: 0.15 | |
| density: 1.0 | |
| lora_rank: 128 | |
| - model: CultriX/Qwen2.5-14B-DeepSeek_r128 | |
| parameters: | |
| weight: 0.15 | |
| density: 1.0 | |
| lora_rank: 128 | |
| # (Optionally, if you wish to “re-add” a full instruct copy you could include it here | |
| # with a higher weight—but note that Qwen2.5-14B-Instruct is already the base.) | |
| - model: Qwen/Qwen2.5-14B-Instruct | |
| parameters: | |
| weight: 0.40 | |
| density: 1.0 | |
| # Merging method and overall parameters | |
| merge_method: dare_ties # Ties corresponding weights across sources. | |
| parameters: | |
| weight: 1.0 # Overall scaling factor. | |
| density: 1.0 # Overall density (typically left at 1.0). | |
| normalize: true # Normalize each set of weights before merging. | |
| int8_mask: true # Enable masking if using int8 quantized weights. | |
| # Use the instruct tokenizer to ensure compatibility. | |
| tokenizer_source: CultriX/Qwen2.5-14B_Virtuoso-small-v2-LoRA_r128 | |
| # Data type for merged weights. | |
| dtype: bfloat16 | |