Add bountybench

app.py

@@ -110,7 +110,7 @@ head_style) as demo:
                 )
                 s.headers = s.check_box['essential'] + s.checkbox_group.value
 
-                if benchmark not in ["SWE-bench-verified", "CyberGym"]:
+                if benchmark not in ["SWE-bench-verified", "CyberGym", "BountyBench"]:
                     with gr.Row():
                         s.model_name = gr.Textbox(
                             value='Input the Model Name (fuzzy, case insensitive)', 
@@ -137,7 +137,7 @@ head_style) as demo:
                     s = structs[benchmark_list.index(dataset_name)]
                     headers = s.check_box['essential'] + fields
                     df = cp.deepcopy(s.table)
-                    if dataset_name not in ["SWE-bench-verified", "CyberGym"]:
+                    if dataset_name not in ["SWE-bench-verified", "CyberGym", "BountyBench"]:
                         default_val = 'Input the Model Name (fuzzy, case insensitive)'
                     else:
                         default_val = 'Input the Agent Name (fuzzy, case insensitive)'
@@ -145,7 +145,7 @@ head_style) as demo:
                     if model_name != default_val:
                         print(model_name)
                         model_name = model_name.lower()
-                        if dataset_name not in ["SWE-bench-verified", "CyberGym"]:
+                        if dataset_name not in ["SWE-bench-verified", "CyberGym", "BountyBench"]:
                             method_names = [x.split('</a>')[0].split('>')[-1].lower() for x in df['Model']]
                         else:
                             method_names = [x.split('</a>')[0].split('>')[-1].lower() for x in df['Agent']]

gen_table.py

@@ -54,9 +54,9 @@ def BUILD_L2_DF(results, benchmark):
     model_list=list(set(model_list))
 
     res = defaultdict(list)
-    if benchmark not in ["RedCode","NYU CTF Bench","PrimeVul","SWE-bench-verified","CyberGym"]:
+    if benchmark not in ["RedCode","NYU CTF Bench","PrimeVul","SWE-bench-verified","CyberGym", "BountyBench"]:
         res['Model']=model_list
-    elif benchmark=="SWE-bench-verified" or benchmark=="CyberGym":
+    elif benchmark in ["SWE-bench-verified", "CyberGym", "BountyBench"]:
         res['Agent']=model_list
     elif benchmark == "PrimeVul":
         used=[]
@@ -104,7 +104,7 @@ def BUILD_L2_DF(results, benchmark):
     required_fields = all_fields
 
     check_box = {}
-    if benchmark in ["SWE-bench-verified", "CyberGym"]:
+    if benchmark in ["SWE-bench-verified", "CyberGym", "BountyBench"]:
         check_box['essential'] = ['Agent']
     elif benchmark=='PrimeVul':
         check_box['essential'] = ['Model','Method']

meta_data.py

@@ -91,4 +91,10 @@ LEADERBOARD_MD['CyberGym'] = """This is a large-scale and high-quality cybersecu
 
 Paper: https://arxiv.org/abs/2506.02548 
 Code: https://github.com/sunblaze-ucb/cybergym
+"""
+
+LEADERBOARD_MD['BountyBench'] = """This is a benchmark with 25 systems with complex, real-world codebases, and includes 40 bug bounties that cover 9 of the OWASP Top 10 Risks.
+
+Paper: https://arxiv.org/abs/2505.15216 
+Code: https://github.com/bountybench/bountybench
 """

results.json

@@ -829,6 +829,29 @@
                 "OpenHands + OpenHands-LM-32B": 0.33,
                 "OpenHands + SWE-Gym-32B": 0.07
             }
+        },
+        "BountyBench": {
+            "Detect Success Rate": {
+                "Claude Code": 5,
+                "OpenAI Codex CLI": 5,
+                "C-Agent: Claude 3.7": 5,
+                "C-Agent: Gemini 2.5": 2.5,
+                "C-Agent: GPT-4.1": 0
+            },
+            "Exploit Success Rate": {
+                "Claude Code": 57.5,
+                "OpenAI Codex CLI": 32.5,
+                "C-Agent: Claude 3.7": 67.5,
+                "C-Agent: Gemini 2.5": 40,
+                "C-Agent: GPT-4.1": 55
+            },
+            "Patch Success Rate": {
+                "Claude Code": 87.5,
+                "OpenAI Codex CLI": 90,
+                "C-Agent: Claude 3.7": 60,
+                "C-Agent: Gemini 2.5": 45,
+                "C-Agent: GPT-4.1": 50
+            }
         }
     }
 }