Upload from GitHub Actions: updated frontend and backend to fix bugs

datasets.json

@@ -219,7 +219,7 @@
         "parallel": true,
         "translation": "machine",
         "base": "MMLU",
-        "implemented": true,
+        "implemented": false,
         "group": "Multitask Language Understanding"
     },
     {
@@ -256,7 +256,7 @@
         "parallel": true,
         "translation": "machine",
         "base": "MMLU",
-        "implemented": true,
+        "implemented": false,
         "group": "Multitask Language Understanding"
     },
     {
@@ -360,7 +360,7 @@
         "parallel": true,
         "translation": "machine",
         "base": "AI2 ARC",
-        "implemented": true,
+        "implemented": false,
         "group": "ARC Question Answering"
     },
     {
@@ -375,7 +375,7 @@
         "parallel": true,
         "translation": "machine",
         "base": "AI2 ARC",
-        "implemented": true,
+        "implemented": false,
         "group": "ARC Question Answering"
     },
     {
@@ -420,7 +420,7 @@
         "parallel": true,
         "translation": "machine",
         "base": "TruthfulQA",
-        "implemented": true,
+        "implemented": false,
         "group": "Truthfulness"
     },
     {
@@ -435,7 +435,7 @@
         "parallel": true,
         "translation": "machine",
         "base": "TruthfulQA",
-        "implemented": true,
+        "implemented": false,
         "group": "Truthfulness"
     },
     {

evals/backend.py

@@ -4,7 +4,7 @@ import os
 import numpy as np
 import pandas as pd
 import uvicorn
-from countries import make_country_table
+from .countries import make_country_table
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
@@ -45,16 +45,25 @@ def compute_normalized_average(df, metrics):
     return normalized_df.mean(axis=1, skipna=False)
 
 
-def make_model_table(df, models):
+def make_model_table(scores_df, models):
     # Create a combined task_metric for origin
-    df["task_metric_origin"] = df["task"] + "_" + df["metric"] + "_" + df["origin"]
-    
+    scores_df["task_metric_origin"] = (
+        scores_df["task"] + "_" + scores_df["metric"] + "_" + scores_df["origin"]
+    )
+
     # Pivot to get scores for each origin-specific metric
-    scores_pivot = df.pivot_table(index="model", columns="task_metric_origin", values="score", aggfunc="mean")
-    
+    scores_pivot = scores_df.pivot_table(
+        index="model",
+        columns="task_metric_origin",
+        values="score",
+        aggfunc="mean",
+    )
+
     # Create the regular task_metric for the main average calculation
-    df["task_metric"] = df["task"] + "_" + df["metric"]
-    main_pivot = df.pivot_table(index="model", columns="task_metric", values="score", aggfunc="mean")
+    scores_df["task_metric"] = scores_df["task"] + "_" + scores_df["metric"]
+    main_pivot = scores_df.pivot_table(
+        index="model", columns="task_metric", values="score", aggfunc="mean"
+    )
 
     # Merge the two pivots
     df = pd.merge(main_pivot, scores_pivot, on="model", how="outer")
@@ -64,6 +73,29 @@ def make_model_table(df, models):
             df[metric] = np.nan
             
     df["average"] = compute_normalized_average(df, task_metrics)
+
+    # Compute origin presence per model+metric
+    origin_presence = (
+        scores_df.groupby(["model", "task_metric", "origin"]).size().unstack(fill_value=0)
+    )
+    # Add boolean flags: show asterisk only if exclusively machine-origin contributed
+    for metric in task_metrics:
+        human_col_name = "human" if "human" in origin_presence.columns else None
+        machine_col_name = "machine" if "machine" in origin_presence.columns else None
+        if human_col_name or machine_col_name:
+            flags = []
+            for model in df.index:
+                try:
+                    counts = origin_presence.loc[(model, metric)]
+                except KeyError:
+                    flags.append(False)
+                    continue
+                human_count = counts.get(human_col_name, 0) if human_col_name else 0
+                machine_count = counts.get(machine_col_name, 0) if machine_col_name else 0
+                flags.append(machine_count > 0 and human_count == 0)
+            df[f"{metric}_is_machine"] = flags
+        else:
+            df[f"{metric}_is_machine"] = False
     df = df.sort_values(by="average", ascending=False).reset_index()
     df = pd.merge(df, models, left_on="model", right_on="id", how="left")
     df["rank"] = df.index + 1
@@ -82,16 +114,25 @@ def make_model_table(df, models):
     return df
 
 
-def make_language_table(df, languages):
+def make_language_table(scores_df, languages):
     # Create a combined task_metric for origin
-    df["task_metric_origin"] = df["task"] + "_" + df["metric"] + "_" + df["origin"]
-    
+    scores_df["task_metric_origin"] = (
+        scores_df["task"] + "_" + scores_df["metric"] + "_" + scores_df["origin"]
+    )
+
     # Pivot to get scores for each origin-specific metric
-    scores_pivot = df.pivot_table(index="bcp_47", columns="task_metric_origin", values="score", aggfunc="mean")
-    
+    scores_pivot = scores_df.pivot_table(
+        index="bcp_47",
+        columns="task_metric_origin",
+        values="score",
+        aggfunc="mean",
+    )
+
     # Create the regular task_metric for the main average calculation
-    df["task_metric"] = df["task"] + "_" + df["metric"]
-    main_pivot = df.pivot_table(index="bcp_47", columns="task_metric", values="score", aggfunc="mean")
+    scores_df["task_metric"] = scores_df["task"] + "_" + scores_df["metric"]
+    main_pivot = scores_df.pivot_table(
+        index="bcp_47", columns="task_metric", values="score", aggfunc="mean"
+    )
 
     # Merge the two pivots
     df = pd.merge(main_pivot, scores_pivot, on="bcp_47", how="outer")
@@ -101,6 +142,36 @@ def make_language_table(df, languages):
             df[metric] = np.nan
             
     df["average"] = compute_normalized_average(df, task_metrics)
+
+    # Compute origin presence per language+metric; show asterisk only if exclusively machine-origin
+    origin_presence = (
+        scores_df.groupby(["bcp_47", "task_metric", "origin"]).size().unstack(fill_value=0)
+    )
+    for metric in task_metrics:
+        human_col_name = "human" if "human" in origin_presence.columns else None
+        machine_col_name = "machine" if "machine" in origin_presence.columns else None
+        if human_col_name or machine_col_name:
+            flags = []
+            for bcp in df.index:
+                try:
+                    counts = origin_presence.loc[(bcp, metric)]
+                except KeyError:
+                    flags.append(False)
+                    continue
+                human_count = counts.get(human_col_name, 0) if human_col_name else 0
+                machine_count = counts.get(machine_col_name, 0) if machine_col_name else 0
+                flags.append(machine_count > 0 and human_count == 0)
+            df[f"{metric}_is_machine"] = flags
+        else:
+            df[f"{metric}_is_machine"] = False
+
+    # Per-row machine-origin flags for each metric (true if any machine-origin score exists for the language)
+    for metric in task_metrics:
+        machine_col = f"{metric}_machine"
+        if machine_col in df.columns:
+            df[f"{metric}_is_machine"] = df[machine_col].notna()
+        else:
+            df[f"{metric}_is_machine"] = False
     df = pd.merge(languages, df, on="bcp_47", how="outer")
     df = df.sort_values(by="speakers", ascending=False)
     

evals/datasets_/mmlu.py

@@ -165,49 +165,55 @@ async def load_mmlu(language_bcp_47, nr):
         return "CohereForAI/Global-MMLU", task, "human"
     elif language_bcp_47 in tags_mmlu_autotranslated:
         ds = _load_dataset("fair-forward/mmlu-autotranslated", language_bcp_47)
-        task = ds["test"].filter(lambda x: x["subject"] == category)[nr]
-        return "fair-forward/mmlu-autotranslated", task, "machine"
+        filtered = ds["test"].filter(lambda x: x["subject"] == category)
+        if nr < len(filtered):
+            task = filtered[nr]
+            return "fair-forward/mmlu-autotranslated", task, "machine"
+        # Requested index exceeds stored sample count → fallback to on-the-fly
+        return await load_mmlu_translated(language_bcp_47, nr)
     else:
-        # Try on-the-fly translation for missing languages
+        # Fallback to on-the-fly translation for missing languages
         return await load_mmlu_translated(language_bcp_47, nr)
 
 
 async def load_mmlu_translated(language_bcp_47, nr):
     """
-    Load MMLU data with on-the-fly Google translation for languages 
-    without native MMLU translations.
+    Load MMLU data with on-the-fly Google translation for languages
+    without native or stored auto-translated MMLU, or when more samples are requested.
     """
-    # Check if Google Translate supports this language
     supported_languages = get_google_supported_languages()
     if language_bcp_47 not in supported_languages:
         return None, None, None
-    
+
     print(f"🔄 Translating MMLU data to {language_bcp_47} on-the-fly...")
-    
+
     try:
-        # Load English MMLU data
+        # Load English MMLU base (AfriMMLU English split for category alignment)
         category = categories[nr % len(categories)]
         ds = _load_dataset("masakhane/afrimmlu", "eng")
         ds = ds.map(parse_choices)
-        task = ds["test"].filter(lambda x: x["subject"] == category)[nr]
-        
+        filtered = ds["test"].filter(lambda x: x["subject"] == category)
+        if len(filtered) == 0:
+            return None, None, None
+        task = filtered[nr % len(filtered)]
+
         # Translate question and choices
         question_translated = await translate_google(task["question"], "en", language_bcp_47)
         choices_translated = []
         for choice in task["choices"]:
             choice_translated = await translate_google(choice, "en", language_bcp_47)
             choices_translated.append(choice_translated)
-        
+
         # Create translated task
         translated_task = {
             "question": question_translated,
             "choices": choices_translated,
             "answer": task["answer"],  # Keep original answer index
-            "subject": task["subject"]
+            "subject": task["subject"],
         }
-        
+
         return f"mmlu-translated-{language_bcp_47}", translated_task, "machine"
-        
+
     except Exception as e:
         print(f"❌ Translation failed for {language_bcp_47}: {e}")
         return None, None, None
@@ -217,7 +223,7 @@ def translate_mmlu(languages):
     human_translated = [*tags_afrimmlu.keys(), *tags_global_mmlu.keys()]
     untranslated = [
         lang
-        for lang in languages["bcp_47"].values[:100]
+        for lang in languages["bcp_47"].values[:150]
         if lang not in human_translated and lang in get_google_supported_languages()
     ]
     n_samples = 10

evals/datasets_/truthfulqa.py

@@ -35,7 +35,7 @@ async def load_truthfulqa(language_bcp_47, nr):
         task = ds["test"][nr]
         return "masakhane/uhura-truthfulqa", task, "human"
     else:
-        # Fallback to on-the-fly translation
+        # Fallback to on-the-fly translation for missing languages/samples
         return await load_truthfulqa_translated(language_bcp_47, nr)
 
 async def load_truthfulqa_translated(language_bcp_47, nr):
@@ -79,10 +79,10 @@ def translate_truthfulqa(languages):
     human_translated = [*tags_uhura_truthfulqa.keys()]
     untranslated = [
         lang
-        for lang in languages["bcp_47"].values[:100]
+        for lang in languages["bcp_47"].values[:150]
         if lang not in human_translated and lang in get_google_supported_languages()
     ]
-    n_samples = 10
+    n_samples = 20
 
     slug = "fair-forward/truthfulqa-autotranslated"
     for lang in tqdm(untranslated):
@@ -132,7 +132,7 @@ def translate_truthfulqa(languages):
                     token=os.getenv("HUGGINGFACE_ACCESS_TOKEN"),
                 )
                 ds_lang.to_json(
-                    f"data/translations/mmlu/{lang}_{split}.json",
+                    f"data/translations/truthfulqa/{lang}_{split}.json",
                     lines=False,
                     force_ascii=False,
                     indent=2,

frontend/src/components/LanguageTable.js

@@ -172,7 +172,7 @@ const LanguageTable = ({ data, selectedLanguages, setSelectedLanguages, totalMod
         filterElement={familyRowFilterTemplate}
         style={{ minWidth: '10rem' }}
       />
-      {ScoreColumns}
+      {ScoreColumns()}
     </DataTable>
   )
 }

frontend/src/components/ScoreColumns.js

@@ -6,7 +6,13 @@ const scoreBodyTemplate = (field, options = {}) => {
 
   return rowData => {
     const score = rowData[field]
-    const isMachineTranslated = machineTranslatedMetrics.includes(field)
+    // Prefer per-row flag if present (backend sets `<metric>_is_machine`),
+    // otherwise fall back to global list
+    const rowFlagKey = `${field}_is_machine`
+    const hasRowFlag = Object.prototype.hasOwnProperty.call(rowData, rowFlagKey)
+    const isMachineTranslated = hasRowFlag
+      ? !!rowData[rowFlagKey]
+      : machineTranslatedMetrics.includes(field)
     return ScoreField(score, minScore, maxScore, isMachineTranslated)
   }
 }

system_architecture_diagram.md

@@ -36,9 +36,9 @@ flowchart TD
     %% On-the-fly Translation with Origin Tagging
     subgraph OTF [On-the-fly Dataset Translation]
         direction LR
-        DS_raw["Raw English Dataset<br/>(e.g., MMLU)"] --> Google_Translate["Google Translate API"]
-        Google_Translate --> DS_translated["Translated Dataset<br/>(e.g., German MMLU)<br/>Origin: 'machine'"]
-        DS_native["Native Dataset<br/>(e.g., German MMLU)<br/>Origin: 'human'"]
+        DS_raw["Raw English Dataset<br/>"] --> Google_Translate["Google Translate API"]
+        Google_Translate --> DS_translated["Translated Dataset<br/>(e.g., MGSM/ARC)<br/>Origin: 'machine'"]
+        DS_native["Native Dataset<br/>(e.g., AfriMMLU/Global-MMLU)<br/>Origin: 'human'"]
     end
     
     %% Evaluation Pipeline
@@ -51,9 +51,9 @@ flowchart TD
     %% Task Execution with Origin Tracking
     P --> Q1[translate_and_evaluate<br/>Origin: 'human']
     P --> Q2[classify_and_evaluate<br/>Origin: 'human']
-    P --> Q3[mmlu_and_evaluate<br/>Origin: 'human'/'machine']
+    P --> Q3[mmlu_and_evaluate<br/>Origin: 'human' (no on-the-fly for missing; uses auto-translated dataset if available)]
     P --> Q4[arc_and_evaluate<br/>Origin: 'human'/'machine']
-    P --> Q5[truthfulqa_and_evaluate<br/>Origin: 'human'/'machine']
+    P --> Q5[truthfulqa_and_evaluate<br/>Origin: 'human' (no on-the-fly for missing; relies on available datasets)]
     P --> Q6[mgsm_and_evaluate<br/>Origin: 'human'/'machine']
     
     %% API Calls with Error Handling
@@ -85,7 +85,7 @@ flowchart TD
     %% Data Sources with Origin Information
     subgraph DS ["Data Sources"]
         DS1["Flores-200<br/>Translation Sentences<br/>Origin: 'human'"]
-        DS2["MMLU/AfriMMLU<br/>Knowledge QA<br/>Origin: 'human'"]
+     DS2["MMLU/AfriMMLU/Global-MMLU<br/>Knowledge QA<br/>Origin: 'human' or 'machine' (HF auto-translated only)"]
         DS3["ARC<br/>Science Reasoning<br/>Origin: 'human'"]
         DS4["TruthfulQA<br/>Truthfulness<br/>Origin: 'human'"]
         DS5["MGSM<br/>Math Problems<br/>Origin: 'human'"]
@@ -97,7 +97,7 @@ flowchart TD
     DS4 --> Q5
     DS5 --> Q6
     
-    DS_translated --> Q3
+     %% No on-the-fly DS_translated for MMLU anymore; only HF auto-translated used
     DS_translated --> Q4
     DS_translated --> Q5