{
"measurement": {
"model.layers.0.self_attn": [
{
"accuracy": 0.8441184080745044,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.8644164686924533,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.8870886344658702,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9216785111317509,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.923924993331495,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9255169967287465,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9412792146598038,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9439142316971955,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9518696005014997,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9535812953192937,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9627055698319485,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9658805839717388,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9671324860108526,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9705542951430145,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9809148741493884,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850361282005906,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839895596041491,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917616180242285,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955931244246465,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.0.mlp": [
{
"accuracy": 0.8021595909407264,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.8082319811770791,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.8396904739110094,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.8496465467308697,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9021237112189594,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9101429554192644,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9236086099163482,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9485961387031957,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.953601500117465,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9501301348209381,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9564386657194087,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9744763845871938,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9782292171332397,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860044463880753,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869898688048124,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900144765790748,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961534016888196,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.1.self_attn": [
{
"accuracy": 0.8832647153421452,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.8911316771256297,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9030310860589931,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9335117726341674,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9408590305400522,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9399805198374548,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9564920957935483,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9553112083751905,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9611115198778479,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9635197706520557,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685047238476967,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9707677925103589,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9712426384028635,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9739498894073462,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826095798788103,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985965903260206,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853341064268821,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931787975916737,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963429286879929,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.1.mlp": [
{
"accuracy": 0.9497420813300108,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9668263519476903,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9657032674080447,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9665720514757068,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862247120441967,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883853314738524,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.990209012267817,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928417815826833,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938126006653827,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935930427286381,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943939370323757,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965882502729073,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968359518649155,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979182915688541,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982497519108859,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986205434512445,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991651891644946,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.2.self_attn": [
{
"accuracy": 0.9844955946318805,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842749297324764,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870261275577114,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989434951560964,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913740343120145,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916360778366461,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939677889801954,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943155316684983,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994640860767839,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948530385675105,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945189355455927,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948051539004633,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961724462796395,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964549103751779,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978278192789539,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981461821681525,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982152157983939,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991599983423887,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995409067479685,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.2.mlp": [
{
"accuracy": 0.967515125037416,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685034234762976,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729101888247227,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742722687262454,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835103846547243,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848232508244875,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867381848100769,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913829983775749,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921353788874847,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915812894898025,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926222521983283,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957083040162137,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963196405013534,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976565420511179,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978053670952489,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981907551777304,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993737504639859,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.3.self_attn": [
{
"accuracy": 0.9695295123383403,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.972145030814174,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9757926966995001,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9789660305372978,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985277837779569,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852157134170595,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99194121771296,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921133655098904,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927570341449035,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928943005047346,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926071238924602,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929811369725748,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947814602208765,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952397996829333,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971521598666808,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975856510634338,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977802630225932,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989008852541079,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994091951000308,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.3.mlp": [
{
"accuracy": 0.9561473143924224,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9574348372652343,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9635815662086794,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9654349223465511,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9777295192899672,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794598124235084,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.982091417315563,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884038954532068,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893846942945138,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.988659122640169,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900293556826287,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942305303962999,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950337114216956,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968633992830291,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970642198489881,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975967746148328,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991703131113594,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.4.self_attn": [
{
"accuracy": 0.9682014531112815,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9689753848177037,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9714684165327957,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771614335967522,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830523770008432,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831419207008654,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900569644462514,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901790620857164,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909771767837044,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913573445335618,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908920872074208,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917225784156471,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932453229312638,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937764226311916,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962328678781265,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968409354332834,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972586136887243,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984311699640545,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992688955297988,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.4.mlp": [
{
"accuracy": 0.9441221882834246,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9459227162756418,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9546413433394934,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9571533563889956,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.971898915442197,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9741261110788113,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776778237305974,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853507618566877,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865835473480585,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856818977400268,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874412269380531,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927095145624327,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937408087211416,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960108565497457,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962875522660876,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970331742599803,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998942600301234,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.5.self_attn": [
{
"accuracy": 0.9613663588503474,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9637175437651182,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9674108421528026,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9733337325681197,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.980797570983046,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.981460536413483,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891694893962458,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895081792124792,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902346635139302,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907651405879542,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905610380351151,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912201200097212,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924562149436066,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930188027945789,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995941169180074,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965408804679388,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971593676661876,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981543259866732,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992503665118037,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.5.mlp": [
{
"accuracy": 0.9345143883440056,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.936705257939665,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9468776099383831,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9497674609485426,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9671208108530233,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9697505062642066,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9738849108842643,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829020056824543,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843033239441482,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832242102803368,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853085259612846,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914528839657769,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926735242632659,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953170131377288,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956259870117432,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964800509072742,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987491455762402,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.6.self_attn": [
{
"accuracy": 0.9552005311768306,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9567459584458878,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9626317602631292,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9716258812695742,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9780041831790617,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783557096594259,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880262972836039,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883122194411331,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890984718099629,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895859425397295,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892202355702849,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897007304978999,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911795917310213,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918636632102885,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952562064128486,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959490506076499,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996896635967699,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997910151788107,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991707398179355,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.6.mlp": [
{
"accuracy": 0.9283958704848039,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9306317369796728,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9419341399089286,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9451246365512672,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9639912726063478,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9668520049829232,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9714217895739957,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812864144577792,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828060740654013,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9816247041484243,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838676502879121,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906269626348818,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919508971077832,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948891131329889,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952233766548728,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961920929950123,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986522533397778,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.7.self_attn": [
{
"accuracy": 0.9501921069857321,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9513117399458823,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9574314506822511,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.967937048916754,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.974776874541452,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9751973222254923,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862523150973415,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864906928943176,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875027630968314,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882083594750025,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874074214726294,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882831246298003,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896295065011241,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904567167702082,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994368409505114,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952011973548093,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964203974967333,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974891248857602,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990458067776146,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.7.mlp": [
{
"accuracy": 0.9245920353814175,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9268923727305312,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9390033864857334,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9424240375427824,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9618747708828825,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9649867642866938,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9699043402154195,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9801335598979342,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818046981174695,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804993515442076,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829283876246527,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900427896186317,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914685397065783,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945654332343685,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949195655062795,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959991773847785,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985667485100413,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.8.self_attn": [
{
"accuracy": 0.9479448684540234,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9497754168941787,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9541806921755013,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9658486945554614,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9732185148290897,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9734971564459173,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847752735950053,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850863272471255,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859606879273135,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869182382612244,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863744316573598,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871331369994503,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884756754005426,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989377694474043,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993746587173327,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946908880466301,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960047114934576,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971651070395877,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989442288627431,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.8.mlp": [
{
"accuracy": 0.9228473340994433,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9251830021017476,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9371756945589655,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9405674000123614,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9609391547151303,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9641326230607534,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9690171123825406,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9796860337159351,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813564848154783,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9799976024384561,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825018468361936,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897747540503348,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912573964776177,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944165692493123,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947638806617377,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958150668657924,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985241532558575,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.9.self_attn": [
{
"accuracy": 0.9434027985522622,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9454548600080767,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.949542000987812,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9607900737931854,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9701321188752589,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9706115654032481,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.983310482979409,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836577467963492,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846839219527809,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856478609634858,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845325376985496,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857009830884635,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873754121118078,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884687162396547,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931544634440032,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941983570859424,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956155265191275,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970092521000066,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988184087223521,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.9.mlp": [
{
"accuracy": 0.9210987700835654,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9235051617418465,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9356520395529897,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9391160613219989,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9601181484758854,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.963349717620172,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9683117154299429,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792310951375648,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9809299999553907,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9796134855010008,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9821251817047596,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895909909698132,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.991073514212315,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943109712391895,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946867446170041,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957683214942288,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984976729273943,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.10.self_attn": [
{
"accuracy": 0.9422470852732658,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9443691797358424,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9505233079391091,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9621549959441549,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9709153023401373,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9711412269327986,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833471354734349,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837823203813872,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851112681882161,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.986037549954888,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853838242492393,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861895909630939,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877430558890888,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887470644653628,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933434389041442,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943583668699782,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957313104900286,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969748152040043,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988512685767522,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.10.mlp": [
{
"accuracy": 0.9200042395999557,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9223897761028064,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9346610015552295,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9381607750332669,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9595750734994286,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9628058863117507,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.96785011358167,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9790496325708533,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.980725462107282,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793568029019394,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818913158225385,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894699650130382,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909605334365839,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942714775618362,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946235836311978,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995691273937394,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984835642726945,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.11.self_attn": [
{
"accuracy": 0.944939869701078,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9466910181861175,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9516582921343415,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9625237129236522,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9713804967897502,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718059670474184,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841814682524848,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845657751552368,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854945629884145,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863942172133217,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856219649756033,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866077588184884,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879403112358168,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888780950813701,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935199338452596,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944160810769781,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959149431179914,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970290351683568,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988997820795416,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.11.mlp": [
{
"accuracy": 0.9184163846075535,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9209518038520688,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9334193262222567,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9369913920956223,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9586414285984478,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9619989283382893,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9671153989865592,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784230808972528,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9801504387961406,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9788080210258302,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814050166230452,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891493693239203,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907028879970312,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940615639482674,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944576281122863,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995577946905733,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984350744665502,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.12.self_attn": [
{
"accuracy": 0.9389739180669973,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.941110919670839,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9485800931916425,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.961893915933998,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.969090601821479,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9694588387777147,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827104437802183,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.98309440249087,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841402279899308,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849093175297113,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845761532012961,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854837474168131,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873370501751962,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882773688602212,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931303880937201,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941311124222059,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954880070382435,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968514701743659,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987963781581799,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.12.mlp": [
{
"accuracy": 0.9167209759746727,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9193145817047671,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.932123426151903,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9359066377540952,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9576744203896899,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9612288042706878,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.96649687923491,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776662593021205,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9795617168573173,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783118089268866,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9810405325536665,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.988899877245881,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905110335614729,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938379212797276,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943462547380477,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955372134542191,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983644112393162,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.13.self_attn": [
{
"accuracy": 0.9330552436019245,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9348286243253633,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9426345919307909,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9549166951328516,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9663865261368061,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9671375750515022,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813106568334135,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9816535556394803,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832854931683916,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841268679225131,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834684191614782,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843010641879549,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861929156131258,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873100315024587,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926183299333053,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935837051291999,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952257742958242,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966287499808363,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987171294633299,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.13.mlp": [
{
"accuracy": 0.9133959749811574,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9161044714090071,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9294791111820623,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9333959904156233,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9561800037167574,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9597750344362698,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9652566498047427,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9769731975699726,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9788662527540797,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776137730008677,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.980387874731892,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885597662804159,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901974077189439,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936694516555259,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941791293309316,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953617051513376,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983101378495217,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.14.self_attn": [
{
"accuracy": 0.9329732756473517,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9355980246082732,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9420463166346675,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9552105876960253,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.966385681574282,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9668581592605302,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9805581754955807,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9809621661215236,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822833940484806,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834464082592412,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830686184822729,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.984181068053371,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856578802787944,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868110011723873,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922007237879658,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993403760912387,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949611068678725,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963712323469257,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986607855044942,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.14.mlp": [
{
"accuracy": 0.908188460492774,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9111755362858898,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9253145581797549,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9296784553872912,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9534304824314619,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9572972800386579,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9631644720701795,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9753440476855949,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773958277349409,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.976193845517149,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9791738104663397,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878310695976803,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895870618669218,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.993198194737105,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938226251029655,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951118296992622,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981887489005825,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.15.self_attn": [
{
"accuracy": 0.9304350420440498,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9322632321793782,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.941121321269556,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9552572467609456,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9645428613416458,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9649363356201273,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9802299337951761,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9806677365283433,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820796900281781,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830972819442028,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822008980830249,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834598432246008,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853309633357352,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865283846463028,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918826735627494,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932231418788433,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947915425965268,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965131774936852,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998625771926814,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.15.mlp": [
{
"accuracy": 0.9030423785902952,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9062142562317221,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9215504748648718,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9261328770141852,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9511807973643667,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9551322582717005,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9613962028371661,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.974268783913239,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763649212882707,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750631139368603,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781357818528226,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.987274421918157,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890862126883707,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929390688867945,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935312707859435,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948759009865554,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981128463772811,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.16.self_attn": [
{
"accuracy": 0.9382822538088811,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9416268222818249,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9486928350831333,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9607921117230466,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9691053619608283,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9697356941178441,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9816082841961792,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9824444329071986,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836695799113888,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845702252105663,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845965458815428,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857065711709622,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872566273092831,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881952560604795,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930350215688936,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940878489716468,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952734143742802,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967616221646926,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987276122433526,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.16.mlp": [
{
"accuracy": 0.903449785160391,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9064931271499709,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9216244207008889,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9262740006180186,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9512182379043416,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9551563397245972,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9614708154021125,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9743258228623554,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764609895646572,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750874214560578,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781633225318632,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872848116547653,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890960430677392,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929590314430626,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935308443872553,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948749667719791,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981064398368624,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.17.self_attn": [
{
"accuracy": 0.9428160366063055,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9456186874916679,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9534695862155211,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9636648429655715,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9714246443227718,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9719139551253695,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835013713883726,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841263545186896,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852017187023241,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859141127549504,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855267879504123,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864635552573753,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.98834245933808,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892407595179975,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934904857685691,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946157879220616,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955267739286157,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971994544060803,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988532903279472,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.17.mlp": [
{
"accuracy": 0.9018948438920473,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9049322885510168,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9205942891146007,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9253567086630746,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9504603783747083,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9544643595030433,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.961014125594183,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9738889200318801,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9760793505334541,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9747107121603269,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9778321939765623,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870828679321628,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889280541162742,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928281833417714,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934479291428273,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948307421107433,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980723507775876,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.18.self_attn": [
{
"accuracy": 0.9529587341178405,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9546628554204577,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9603673153017697,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9687020409067995,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758516212337112,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9765019342303276,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853251299477721,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862821217577317,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874321879622968,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880474993321848,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879261279527686,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887461391963849,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990133128937726,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909511174467441,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946067581247342,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995469303715876,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962945206115317,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976073616417125,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99898503124285,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.18.mlp": [
{
"accuracy": 0.9053857495125971,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9081258532640181,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9229096885966628,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.927355877270824,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9522143559236276,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9559730050203047,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9622014600195383,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.974984212043254,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9770735514497286,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9756023371101994,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785740162785116,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875524161432526,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893086236460429,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931733001170582,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936692649498582,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994983588602688,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981764601788631,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.19.self_attn": [
{
"accuracy": 0.9534452263461917,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9549659222952629,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9638146519857018,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9707228834005562,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9766234872176459,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9759080110019759,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854976044673669,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856271768527988,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875179453703918,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880504351305334,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880321815021729,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888056345823172,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906873627026614,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914152236704371,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948624689744687,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957822212753328,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961582149722075,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978303080853564,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989550467733735,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.19.mlp": [
{
"accuracy": 0.9062681023619676,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9089841560313576,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9231530631843367,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.927404076253113,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9525698967474072,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9563319257607585,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9623484199768618,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9752343678846955,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9772863054255906,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758144192593662,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787874992838815,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876855290739944,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.989411055110395,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932321799968026,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937424305336255,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949483710316647,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981841927112424,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.20.self_attn": [
{
"accuracy": 0.9548610912537888,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9572624871997457,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9646144836748901,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9716195778333043,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9768829655980593,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9779445880925969,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860794534906745,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865944413713327,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.98746091316111,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878654490519119,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886093894019723,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891465613373408,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909034950021458,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916583408944701,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950313634533239,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958302603029695,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963590044675297,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978011067172414,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990363206904332,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.20.mlp": [
{
"accuracy": 0.9052868973659842,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9079893352954012,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9221028981632308,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9263538579015356,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9520051195041129,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.955764699335161,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9617814496159554,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749245413235927,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9770274423062801,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755291904960024,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785112825859534,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.987541364554904,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892864228765431,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931938315153515,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936666861176491,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948774828256941,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981871976419107,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.21.self_attn": [
{
"accuracy": 0.9525331523465483,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9543475191059866,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9623144930718761,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9695970120986825,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763068562177452,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9767041819562253,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852536986336896,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861459419082262,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871043979778493,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876837364741062,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880848060313024,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890199949915864,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990567693903454,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912368868183541,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947961357691767,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956646909152991,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961309363624375,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976780728596312,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989820220415775,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.21.mlp": [
{
"accuracy": 0.9059562218424521,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9086698732878032,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9223203978648311,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9264417920065553,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.952298901681053,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9560690135940125,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9618849821976925,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749852822799432,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771094032024082,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9756772655895666,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786557658624492,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876080941850025,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.989350813367453,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.993171383724793,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937113397264559,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948797431945997,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981552784723279,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.22.self_attn": [
{
"accuracy": 0.9564492947569019,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9578231339784045,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9651919899410323,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9723307475643722,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783886268146729,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785928122307125,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863011046361766,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871218914019042,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880379328602239,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886001791854045,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890172332035083,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897766924512229,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912757728689987,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918896999014052,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951134300055472,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959200014556316,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963591141797798,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997846530254097,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990268514589652,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.22.mlp": [
{
"accuracy": 0.9090014011844209,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9115287432152974,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9244405423340044,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.928338359845312,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9538730471149871,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9574429634958506,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9629494696855545,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9759830266335293,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9779725233583074,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9765294037857338,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793511999672965,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880734737472314,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.989716893282572,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935076340885931,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939369862335489,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950198775868079,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982532269419416,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.23.self_attn": [
{
"accuracy": 0.9599294948734736,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9607209261310727,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9692706522580824,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9747190769565733,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792776388165197,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9795575590902253,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868756065047101,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874882602230891,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888317307310277,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.989245100020382,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898174084036758,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905008022477361,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921464296302905,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927376621451817,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995597775192245,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964066172190207,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965246782491082,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981645762822345,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990890388113545,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.23.mlp": [
{
"accuracy": 0.9108333932726007,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9132585923530554,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9254217869357059,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9290946498513222,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9547145624498004,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9582169928440922,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9634277876466513,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764403795431319,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783937334034004,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9769721023836418,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9797303367798266,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883016266762034,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.989903926236653,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936299696074504,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940533732725797,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950694290210346,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998276997014488,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.24.self_attn": [
{
"accuracy": 0.957895180504573,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9594362385963139,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9672121193848158,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729409372728122,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9778250600456407,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784931324814495,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861810530878996,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.986898737322343,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880443615348715,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884597778712448,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889824723236655,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989470063880282,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991407775736757,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921262290230707,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952057520613858,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960664976143131,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961930544630281,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979532692936788,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990140444378516,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.24.mlp": [
{
"accuracy": 0.9119349463205588,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9143685765172306,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9259382256943929,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9294842967861577,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9551709153150257,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9586779156601742,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9636854397034958,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9766589038466152,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786053893009299,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.977162377663741,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9799375719831962,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.988381516384451,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900074740381617,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936663947362256,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940822284521633,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995053324774888,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998282640614841,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.25.self_attn": [
{
"accuracy": 0.9571951864973495,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9588290264731959,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9664914594276955,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9720899252907226,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9777896461616221,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783101594075561,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856845156142586,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.986773371941557,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878869317471981,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.988309536619406,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888341469003966,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896434013437676,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913406613969097,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922596530645693,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952919731088179,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961415890138596,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962817403659421,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978995825972801,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989976871936386,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.25.mlp": [
{
"accuracy": 0.9118195711389968,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9142580883283364,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9255682288816101,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.929041293302649,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9551029095524236,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9585770667579613,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.963450997577686,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9765840904591115,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785386049433759,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771397227029267,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798936337036522,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.988373677417832,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899806563350323,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936594816512967,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.994089132652765,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950306218999782,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982861557264665,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.26.self_attn": [
{
"accuracy": 0.9562748840176746,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9574169062666202,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9651306679374293,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9719903807302839,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774152503201836,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.978190628909751,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859712653744378,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866929001321918,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875935172711156,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881195105661295,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884453238184122,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892201295465624,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99104388229745,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917406401057777,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949950118990321,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958232179602706,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961928661643086,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977652271290457,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990028600043029,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.26.mlp": [
{
"accuracy": 0.9112869205051347,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9137654928000349,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9250450955801889,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9285375728811088,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9548271270959001,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9583619855540364,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9632232696994355,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764096584955329,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783854971786863,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9770015732042099,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9797752542225154,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883054592891743,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899194352818947,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936001463303048,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940526827459076,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950002818981087,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998265868690061,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.27.self_attn": [
{
"accuracy": 0.9506613469045413,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9516085662731999,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9615890748406711,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685771850574958,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755438011825869,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758185616351271,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838469578933559,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849262821341032,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861576297369442,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867623732180187,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.987944562625336,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885601222907242,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901024750924032,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908893148316756,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945063135635696,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954511554740173,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957854168268999,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974954438665392,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988790563871398,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.27.mlp": [
{
"accuracy": 0.90896334754009,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9115500944225412,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9229131407643619,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9265510510457189,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9535836806814921,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9572188114061168,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9621657284074708,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9756415339774991,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9777122934005762,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763527628230421,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792174773878957,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879873472167865,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896446707609453,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934007478142647,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938996430412915,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948689580012701,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982173452889057,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.28.self_attn": [
{
"accuracy": 0.9448457772010251,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9466259624612958,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9572204351425171,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9664924842746634,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9721318898036292,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9723922438723477,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836740098697575,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842817011338315,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.985425343982091,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860875624673147,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858612877463824,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869970555270189,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892060965495674,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900840205924684,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940992693514809,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994968607261973,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957085496035257,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973991556203385,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988291055656431,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.28.mlp": [
{
"accuracy": 0.9042005593839445,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9069336727261543,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9188475273549557,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9227865683405023,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9511604931597647,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9549492048589807,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9601643316839871,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742505496956015,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764398665804612,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9751316304073522,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781226874574235,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873589961789548,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890993560517305,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930277266539633,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935826998767688,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994618469003686,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981034057901094,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.29.self_attn": [
{
"accuracy": 0.9461272960822833,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9485039587475752,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9583730721159985,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9655470142239019,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9722677784922876,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9731917275409949,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838995892358454,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845045938793766,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856399943267828,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861191138625145,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.986349486549826,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.987041790626551,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893495736055469,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903936231214749,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943056387802292,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952229492145738,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957427866266746,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975195012214643,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988640583653347,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.29.mlp": [
{
"accuracy": 0.8984877557346695,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9014385948447805,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.914039778670198,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9182223722730812,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9477840436524466,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9521516572291914,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9577506119875532,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9723792723134944,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9747737388273603,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9732215984871513,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9766073584752647,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863008428855162,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883121864772156,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992336943843647,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930485067644009,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941539493585495,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979229370119834,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.30.self_attn": [
{
"accuracy": 0.9369314536452293,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9403441822842548,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9515306748645871,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9602761572520984,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9687328972902737,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9690217930627497,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811916990009578,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818324508439553,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834755148346487,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843322616070509,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848656471034414,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857339646275106,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882232345346558,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890903405796149,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934890108232043,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945925705036834,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99507573504295,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970997079385837,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986809876036683,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.30.mlp": [
{
"accuracy": 0.8880669466759029,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.8918650973784297,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9046222469524333,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9087879198946451,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.941326686230145,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9474482479456224,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9532236438244581,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9694917461785831,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9720300884035072,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9697374729929786,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742595873969165,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844932384592923,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871056042121429,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912974776111936,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920521853491664,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931766462120178,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975960793794671,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.31.self_attn": [
{
"accuracy": 0.9343558506746041,
"total_bits": 89665536,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9387940903635401,
"total_bits": 92221440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.950289665378238,
"total_bits": 95758848,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9634017239471799,
"total_bits": 112272384,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685949000873064,
"total_bits": 132913152,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9690707978841505,
"total_bits": 132980224,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804592104628682,
"total_bits": 169613312,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.981143138322391,
"total_bits": 169745920,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825306637212634,
"total_bits": 171195392,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833268088925826,
"total_bits": 173563904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841943790057772,
"total_bits": 174923264,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851906188812695,
"total_bits": 175750144,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873950573566713,
"total_bits": 179253248,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882553996469238,
"total_bits": 181592064,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931684818952099,
"total_bits": 220469248,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940987521856043,
"total_bits": 223535104,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948879069109496,
"total_bits": 253499392,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996840799386662,
"total_bits": 265838592,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986135794407394,
"total_bits": 337385472,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.31.mlp": [
{
"accuracy": 0.8713396305316373,
"total_bits": 395461696,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.873587169145283,
"total_bits": 409224256,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.8843217009776517,
"total_bits": 457272320,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.8881954195860186,
"total_bits": 512977920,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9393127135148174,
"total_bits": 578397280,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9460893338055987,
"total_bits": 592872448,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9510163431496996,
"total_bits": 637454432,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9686275648051187,
"total_bits": 728741472,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9714018830441331,
"total_bits": 739476480,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.969087376896488,
"total_bits": 751543392,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.973751128259066,
"total_bits": 766018560,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839688859585869,
"total_bits": 924689504,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866706707122687,
"total_bits": 939164672,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908757904593489,
"total_bits": 1069524064,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917125939881724,
"total_bits": 1110384896,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992747723357752,
"total_bits": 1209999616,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972542343097494,
"total_bits": 1415520512,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.norm.norm": null,
"lm_head.linear": null
},
"last_module_idx": 66
}