new metrics

src/display/utils.py

@@ -252,7 +252,7 @@ class GuardBenchColumn:
         name="jailbreaked_answers_f1",
         display_name="Jailbreaked Answers F1",
         type="number",
-        displayed_by_default=True
+        displayed_by_default=False
     ))
     jailbreaked_answers_recall_binary: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="jailbreaked_answers_recall_binary",
@@ -278,6 +278,12 @@ class GuardBenchColumn:
         type="number",
         displayed_by_default=False
     ))
+    integral_score: ColumnInfo = field(default_factory=lambda: ColumnInfo(
+        name="integral_score",
+        display_name="Integral Score",
+        type="number",
+        displayed_by_default=True
+    ))
 
     # Calculated overall metrics (renamed)
     macro_accuracy: ColumnInfo = field(default_factory=lambda: ColumnInfo(
@@ -298,12 +304,7 @@ class GuardBenchColumn:
         type="number",
         displayed_by_default=False
     ))
-    integral_score: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="integral_score",
-        display_name="Integral Score",
-        type="number",
-        displayed_by_default=True
-    ))
+
     # NEW Summary Metrics
     micro_avg_error_ratio: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="micro_avg_error_ratio",
@@ -367,7 +368,8 @@ METRICS = [
     "recall_binary",
     "precision_binary",
     "error_ratio",
-    "avg_runtime_ms"
+    "avg_runtime_ms",
+    "accuracy"
 ]
 
 def get_all_column_choices():

src/leaderboard/processor.py

@@ -19,52 +19,59 @@ MAX_RUNTIME_PENALTY = 0.75 # Corresponds to 1.0 - MIN_TIME_FACTOR, library used
 def calculate_integral_score(row: pd.Series) -> float:
     """
     Calculate the integral score for a given model entry row.
-    Uses F1-binary as the primary metric, error ratio, and runtime penalty.
+    Uses accuracy as the primary metric, micro error ratio, and micro runtime penalty.
+    Falls back to macro accuracy and averaged per-test-type errors/runtimes if micro values are missing.
     """
     integral_score = 1.0
     metric_count = 0
 
-    # Primary metric (using f1_binary, could be changed to accuracy if needed)
+    # Primary metric (using accuracy)
     for test_type in TEST_TYPES:
-        metric_col = f"{test_type}_f1_binary"
+        metric_col = f"{test_type}_accuracy"
         if metric_col in row and pd.notna(row[metric_col]):
             integral_score *= row[metric_col]
             metric_count += 1
 
-    # If no primary metrics found, return 0
+    # Fallback if no primary metrics found
     if metric_count == 0:
-        # Check for average_f1 as a fallback
-        if "average_f1" in row and pd.notna(row["average_f1"]):
-            integral_score *= row["average_f1"]
+        if "macro_accuracy" in row and pd.notna(row["macro_accuracy"]):
+            integral_score *= row["macro_accuracy"]
             metric_count += 1
         else:
             return 0.0 # Cannot calculate score without primary metrics
 
-    # Account for average errors across all test types (using a simple average for now)
-    # This requires micro-level error data which isn't directly in avg_metrics.
-    # We'll approximate using the average of available error ratios.
-    error_ratios = []
-    for test_type in TEST_TYPES:
-        error_col = f"{test_type}_error_ratio"
-        if error_col in row and pd.notna(row[error_col]):
-            error_ratios.append(row[error_col])
-
-    if error_ratios:
-        avg_error_ratio = np.mean(error_ratios)
-        integral_score *= (1.0 - avg_error_ratio)
-
-    # Account for average runtime across all test types (using a simple average for now)
-    # This requires micro-level runtime data. We'll approximate.
-    runtimes = []
-    for test_type in TEST_TYPES:
-        runtime_col = f"{test_type}_avg_runtime_ms"
-        if runtime_col in row and pd.notna(row[runtime_col]):
-            runtimes.append(row[runtime_col])
-
-    if runtimes:
-        avg_runtime_ms = np.mean(runtimes)
-
-        # Apply penalty based on runtime
+    # Error Penalty
+    micro_error_col = "micro_avg_error_ratio"
+    if micro_error_col in row and pd.notna(row[micro_error_col]):
+        # Micro error is stored as %, convert back to ratio
+        micro_error_ratio = row[micro_error_col] / 100.0
+        integral_score *= (1.0 - micro_error_ratio)
+    else:
+        # Fallback: Calculate average error from per-test-type
+        error_ratios = []
+        for test_type in TEST_TYPES:
+            error_col = f"{test_type}_error_ratio"
+            if error_col in row and pd.notna(row[error_col]):
+                error_ratios.append(row[error_col])
+        if error_ratios:
+            avg_error_ratio = np.mean(error_ratios)
+            integral_score *= (1.0 - avg_error_ratio)
+
+    # Runtime Penalty
+    micro_runtime_col = "micro_avg_runtime_ms"
+    if micro_runtime_col in row and pd.notna(row[micro_runtime_col]):
+        avg_runtime_ms = row[micro_runtime_col]
+    else:
+        # Fallback: Calculate average runtime from per-test-type
+        runtimes = []
+        for test_type in TEST_TYPES:
+            runtime_col = f"{test_type}_avg_runtime_ms"
+            if runtime_col in row and pd.notna(row[runtime_col]):
+                runtimes.append(row[runtime_col])
+        avg_runtime_ms = np.mean(runtimes) if runtimes else None
+
+    if avg_runtime_ms is not None:
+        # Apply penalty based on runtime (using micro or calculated average)
         runtime = max(
             min(avg_runtime_ms, MAX_PUNISHABLE_RUNTIME_MS),
             MIN_PUNISHABLE_RUNTIME_MS,
@@ -76,15 +83,12 @@ def calculate_integral_score(row: pd.Series) -> float:
             )
             time_factor = 1.0 - MAX_RUNTIME_PENALTY * normalized_time
         else:
-            time_factor = 1.0 if runtime <= MIN_PUNISHABLE_RUNTIME_MS else (1.0 - MAX_RUNTIME_PENALTY) # Assign max penalty if runtime exceeds min when max==min
+            time_factor = 1.0 if runtime <= MIN_PUNISHABLE_RUNTIME_MS else (1.0 - MAX_RUNTIME_PENALTY)
 
-        # Make sure the factor is not less than the minimum value (1 - MAX_PENALTY)
         time_factor = max((1.0 - MAX_RUNTIME_PENALTY), time_factor)
         integral_score *= time_factor
 
-    # Root the score by the number of primary metrics used? (Optional, library did this)
-    # return integral_score ** (1 / metric_count) if metric_count > 0 else 0.0
-    # Let's skip the rooting for now to keep the scale potentially larger.
+    # Rooting is not done in the reference library's summary table calculation
     return integral_score
 
 
@@ -210,14 +214,25 @@ def leaderboard_to_dataframe(leaderboard_data: Dict) -> pd.DataFrame:
                                 row[f"{test_type}_f1"] = metrics[metric]
 
             # Calculate averages if not present
+            # Use accuracy for macro_accuracy
             if "macro_accuracy" not in row:
-                f1_values = []
+                accuracy_values = []
                 for test_type in TEST_TYPES:
-                    if test_type in avg_metrics and "f1_binary" in avg_metrics[test_type] and pd.notna(avg_metrics[test_type]["f1_binary"]):
-                        f1_values.append(avg_metrics[test_type]["f1_binary"])
-                if f1_values:
-                    row["macro_accuracy"] = sum(f1_values) / len(f1_values)
+                    # Check avg_metrics structure first
+                    accuracy_val = None
+                    if test_type in avg_metrics and "accuracy" in avg_metrics[test_type] and pd.notna(avg_metrics[test_type]["accuracy"]):
+                        accuracy_val = avg_metrics[test_type]["accuracy"]
+                    # Check flat structure as fallback (might be redundant but safer)
+                    elif f"{test_type}_accuracy" in row and pd.notna(row[f"{test_type}_accuracy"]):
+                        accuracy_val = row[f"{test_type}_accuracy"]
+
+                    if accuracy_val is not None:
+                        accuracy_values.append(accuracy_val)
 
+                if accuracy_values:
+                    row["macro_accuracy"] = sum(accuracy_values) / len(accuracy_values)
+
+            # Use recall_binary for macro_recall
             if "macro_recall" not in row:
                 recall_values = []
                 for test_type in TEST_TYPES:
@@ -255,9 +270,14 @@ def leaderboard_to_dataframe(leaderboard_data: Dict) -> pd.DataFrame:
             col_name = f"{test_type}_{metric}"
             if col_name not in df.columns:
                 df[col_name] = pd.NA # Use pd.NA for missing numeric data
-            # Add non-binary F1 if binary exists
+
+            # Add non-binary F1 if binary exists and f1 is missing
             if metric == "f1_binary" and f"{test_type}_f1" not in df.columns:
-                df[f"{test_type}_f1"] = df[col_name] # Copy f1_binary to f1 if f1 is missing
+                # Check if the binary column has data before copying
+                if col_name in df.columns:
+                    df[f"{test_type}_f1"] = df[col_name]
+                else:
+                    df[f"{test_type}_f1"] = pd.NA
 
     # Calculate Integral Score
     if not df.empty: