Add TurkishMMLU dataset and results (#2)

- Add TurkishMMLU dataset and results (b949e7e3ca62dc6da4c6623dbee14f0cc1843004)

data/datasets.json

@@ -189,5 +189,13 @@
         "url": "https://huggingface.co/datasets/furkanunluturk/turkce-atasozleri",
         "hf_name": "abrek/turkce-atasozleri-lm-evaluation-harness",
         "generative": false
+    },
+    "turkishmmlu": {
+        "name": "TurkishMMLU",
+        "task": "multiple_choice",
+        "description": "TurkishMMLU is a multiple-choice dataset for Turkish Natural Language Processing (NLP) community based on Turkish Highschool Curricula for nine subjects.",
+        "url": "https://huggingface.co/datasets/AYueksel/TurkishMMLU",
+        "hf_name": "AYueksel/TurkishMMLU",
+        "generative": false
     }
 }

results/zero-shot/CerebrumTech__cere-llama-3-8b-tr.json

@@ -174,6 +174,12 @@
             "task": "multiple_choice",
             "acc": 0.48092485549132946,
             "acc_norm": 0.48092485549132946
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.25555555555555554,
+            "acc_norm": 0.25555555555555554
         }
     ]
 }

results/zero-shot/Llama-3.3-70B-Instruct.json

@@ -176,6 +176,12 @@
             "task": "multiple_choice",
             "acc": 0.9254335260115607,
             "acc_norm": 0.9254335260115607
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.646,
+            "acc_norm": 0.646
         }
     ]
 }

results/zero-shot/Ministral-8B-Instruct.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.4046242774566474,
             "acc_norm": 0.4046242774566474
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.2644444444444444,
+            "acc_norm": 0.2644444444444444
         }
     ]
 }

results/zero-shot/Mistral-7B-Instruct-v0.3.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.3,
             "acc_norm": 0.3
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.19555555555555557,
+            "acc_norm": 0.19555555555555557
         }
     ]
 }

results/zero-shot/Mistral-7B-v0.3.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.27572254335260116,
             "acc_norm": 0.27572254335260116
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.2688888888888889,
+            "acc_norm": 0.2688888888888889
         }
     ]
 }

results/zero-shot/Mixtral-8x7B-Instruct-v0.1.json

@@ -175,6 +175,12 @@
             "task": "multiple_choice",
             "acc": 0.5150289017341041,
             "acc_norm": 0.5150289017341041
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.358,
+            "acc_norm": 0.358
         }
     ]
 }

results/zero-shot/Qwen2.5-0.5B-Instruct.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.2832369942196532,
             "acc_norm": 0.2832369942196532
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.2111111111111111,
+            "acc_norm": 0.2111111111111111
         }
     ]
 }

results/zero-shot/Qwen2.5-0.5B.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.20346820809248556,
             "acc_norm": 0.20346820809248556
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.17888888888888888,
+            "acc_norm": 0.17888888888888888
         }
     ]
 }

results/zero-shot/Qwen2.5-1.5B-Instruct.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.3468208092485549,
             "acc_norm": 0.3468208092485549
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.28888888888888886,
+            "acc_norm": 0.28888888888888886
         }
     ]
 }

results/zero-shot/Qwen2.5-1.5B.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.2300578034682081,
             "acc_norm": 0.2300578034682081
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.23,
+            "acc_norm": 0.23
         }
     ]
 }

results/zero-shot/Qwen2.5-14B-Instruct.json

@@ -175,6 +175,12 @@
             "task": "multiple_choice",
             "acc": 0.7832369942196532,
             "acc_norm": 0.7832369942196532
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.5944444444444444,
+            "acc_norm": 0.5944444444444444
         }
     ]
 }

results/zero-shot/Qwen2.5-14B.json

@@ -175,6 +175,12 @@
             "task": "multiple_choice",
             "acc": 0.753757225433526,
             "acc_norm": 0.753757225433526
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.5622222222222222,
+            "acc_norm": 0.5622222222222222
         }
     ]
 }

results/zero-shot/Qwen2.5-3B-Instruct.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.6011560693641619,
             "acc_norm": 0.6011560693641619
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.37777777777777777,
+            "acc_norm": 0.37777777777777777
         }
     ]
 }

results/zero-shot/Qwen2.5-3B.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.4346820809248555,
             "acc_norm": 0.4346820809248555
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.22555555555555556,
+            "acc_norm": 0.22555555555555556
         }
     ]
 }

results/zero-shot/Qwen2.5-7B-Instruct.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.7121387283236994,
             "acc_norm": 0.7121387283236994
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.47555555555555556,
+            "acc_norm": 0.47555555555555556
         }
     ]
 }

results/zero-shot/Qwen2.5-7B.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.7352601156069364,
             "acc_norm": 0.7352601156069364
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.49333333333333335,
+            "acc_norm": 0.49333333333333335
         }
     ]
 }

results/zero-shot/aya-23-35B.json

@@ -175,6 +175,12 @@
             "task": "multiple_choice",
             "acc": 0.5687861271676301,
             "acc_norm": 0.5687861271676301
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.4533333333333333,
+            "acc_norm": 0.4533333333333333
         }
     ]
 }

results/zero-shot/aya-23-8b.json

@@ -169,6 +169,12 @@
             "task": "multiple_choice",
             "acc": 0.44971098265895953,
             "acc_norm": 0.44971098265895953
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.33,
+            "acc_norm": 0.33
         }
     ]
 }

results/zero-shot/aya-expanse-32b.json

@@ -174,6 +174,12 @@
             "task": "multiple_choice",
             "acc": 0.8236994219653179,
             "acc_norm": 0.8236994219653179
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.5688888888888889,
+            "acc_norm": 0.5688888888888889
         }
     ]
 }

results/zero-shot/aya-expanse-8b.json

@@ -160,6 +160,12 @@
             "task": "multiple_choice",
             "acc": 0.723121387283237,
             "acc_norm": 0.723121387283237
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.46555555555555556,
+            "acc_norm": 0.46555555555555556
         }
     ]
 }

results/zero-shot/aya101.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.009826589595375723,
             "acc_norm": 0.009826589595375723
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.374,
+            "acc_norm": 0.374
         }
     ]
 }

results/zero-shot/commencis-7b.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.22658959537572254,
             "acc_norm": 0.22658959537572254
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.24666666666666667,
+            "acc_norm": 0.24666666666666667
         }
     ]
 }

results/zero-shot/kanarya-2b.json

@@ -172,6 +172,12 @@
             "task": "multiple_choice",
             "acc": 0.0,
             "acc_norm": 0.0
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.18,
+            "acc_norm": 0.18
         }
     ]
 }

results/zero-shot/llama-3-8b-instruct.json

@@ -168,6 +168,13 @@
             "task": "multiple_choice",
             "acc": 0.6947976878612717,
             "acc_norm": 0.6947976878612717
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.3811111111111111,
+            "acc_norm": 0.3811111111111111
         }
+
     ]
 }

results/zero-shot/llama-3-8b.json

@@ -167,6 +167,13 @@
             "task": "multiple_choice",
             "acc": 0.44046242774566474,
             "acc_norm": 0.44046242774566474
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.2544444444444444,
+            "acc_norm": 0.2544444444444444
         }
+
     ]
 }

results/zero-shot/llama-3.1-8b-instruct.json

@@ -167,6 +167,13 @@
             "task": "multiple_choice",
             "acc": 0.7549132947976879,
             "acc_norm": 0.7549132947976879
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.3811111111111111,
+            "acc_norm": 0.3811111111111111
         }
+
     ]
 }

results/zero-shot/llama-3.1-8b.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.5410404624277456,
             "acc_norm": 0.5410404624277456
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.3055555555555556,
+            "acc_norm": 0.3055555555555556
         }
     ]
 }

results/zero-shot/llama-3.2-1b.json

@@ -199,6 +199,13 @@
             "task": "multiple_choice",
             "acc": 0.21676300578034682,
             "acc_norm": 0.21676300578034682
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.18888888888888888,
+            "acc_norm": 0.18888888888888888
         }
+
     ]
 }

results/zero-shot/llama-3.2-3b-instruct.json

@@ -192,6 +192,13 @@
             "task": "multiple_choice",
             "acc": 0.010982658959537572,
             "acc_norm": 0.010982658959537572
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.34444444444444444,
+            "acc_norm": 0.34444444444444444
         }
+
     ]
 }

results/zero-shot/llama-3.2-3b.json

@@ -161,6 +161,13 @@
             "task": "multiple_choice",
             "acc": 0.1994219653179191,
             "acc_norm": 0.1994219653179191
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.29,
+            "acc_norm": 0.29
         }
+ 
     ]
 }

results/zero-shot/mistral-7b.json

@@ -166,6 +166,12 @@
             "task": "multiple_choice",
             "acc": 0.30809248554913293,
             "acc_norm": 0.30809248554913293
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.20333333333333334,
+            "acc_norm": 0.20333333333333334
         }
     ]
 }

results/zero-shot/trendyol-7b.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.0,
             "acc_norm": 0.0
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.2477777777777778,
+            "acc_norm": 0.2477777777777778
         }
     ]
 }

results/zero-shot/turna.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.19248554913294796,
             "acc_norm": 0.19248554913294796
+        },
+        {
+            "name": "turkishmmlu",
+            "task": "multiple_choice",
+            "acc": 0.19333333333333333,
+            "acc_norm": 0.19333333333333333
         }
     ]
 }