Update meta_data.py

meta_data.py

@@ -1,88 +1,91 @@
-VLMEVALKIT_README = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/README.md'
-# CONSTANTS-CITATION
-CITATION_BUTTON_TEXT = r"""@article{guo2025sok,
-  title={{Frontier AI's Impact on the Cybersecurity Landscape}},
-  author={Guo, Wenbo and Potter, Yujin and Shi, Tianneng and Wang, Zhun and Zhang, Andy and Song, Dawn},
-  journal={arXiv preprint arXiv:2504.05408},
-  year={2025}
-}
-"""
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-# CONSTANTS-TEXT
-LEADERBORAD_INTRODUCTION = """# Cybersecurity Leaderboard
-### Welcome to the Cybersecurity Leaderboard! This leaderboard is a collection of benchmarks relevant to cybersecurity capabilities. 
-This leaderboard covers {} benchmarks.
-
-This leaderboard was last updated: {} """
-# CONSTANTS-FIELDS
-# META_FIELDS = [
-#     'Model'
-# ]
-
-DEFAULT_TASK = [
-    'Vulnerable Code Generation', 'Attack Generation', 'CTF', 'Cyber Knowledge', 'Pen Test', 'Vulnerability Detection', 'PoC Generation', 'Patching'
-]
-
-# The README file for each benchmark
-LEADERBOARD_MD = {}
-
-LEADERBOARD_MD['CyberSecEval-3'] = """CyberSecEval-3 is a security benchmarks for LLMs. CyberSecEval-3 assesses 8 different risks across two broad categories: risk to third parties, and risk to application developers and end users.  
-
-Paper: https://arxiv.org/abs/2408.01605 
-Code: https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks 
-"""
-LEADERBOARD_MD['SecCodePLT'] = """ SecCodePLT is a unified and comprehensive evaluation platform for code GenAIs' risks. This benchmark consists of insecure coding tasks and cyberattack helpfulness tasks. The helpfulness tasks are designed considering five attack steps: reconnaissance, weaponization & infiltration, C2 & execution, discovery, and collection.
-
-Paper: https://arxiv.org/abs/2410.11096 
-Code: https://github.com/CodeSecPLT/CodeSecPLT 
-"""
-LEADERBOARD_MD['RedCode'] = """RedCode is a benchmark for risky code execution and generation: (1) RedCode-Exec provides challenging prompts that could lead to risky code execution, aiming to evaluate code agents' ability to recognize and handle unsafe code. (2) RedCode-Gen provides 160 prompts with function signatures and docstrings as input to assess whether code agents will follow instructions to generate harmful code or software.
-
-Paper: https://arxiv.org/abs/2411.07781
-Code: https://github.com/AI-secure/RedCode 
-"""
-LEADERBOARD_MD['CyBench'] = """Cybench is a framework for specifying cybersecurity tasks and evaluating agents on those tasks. This includes 40 professional-level Capture the Flag (CTF) tasks from 4 distinct CTF competitions, chosen to be recent, meaningful, and spanning a wide range of difficulties.
-
-Paper: https://arxiv.org/abs/2408.08926 
-Code: https://github.com/andyzorigin/cybench 
-"""
-LEADERBOARD_MD['NYU CTF Bench'] = """This assesses LLMs in solving CTF challenges. This includes a diverse range of CTF challenges from popular competitions.
-
-Paper: https://arxiv.org/abs/2406.05590 
-Code: https://github.com/NYU-LLM-CTF/NYU_CTF_Bench 
-"""
-LEADERBOARD_MD['CyberBench'] = """CyberBench is a multi-task benchmark to evaluate the model knowledge in cybersecurity. 
-
-Paper: https://zefang-liu.github.io/files/liu2024cyberbench_paper.pdf 
-Code: https://github.com/jpmorganchase/CyberBench
-"""
-LEADERBOARD_MD['CyberMetric'] = """CyberMetric is designed to accurately test the general knowledge of LLMs in cybersecurity. CyberMetric-80, CyberMetric-500, CyberMetric-2000, and CyberMetric-10000 are multiple-choice Q&A benchmark datasets comprising 80, 500, 2000, and 10,000 questions, respectively.
-
-Paper: https://arxiv.org/abs/2402.07688 
-Code: https://github.com/cybermetric/CyberMetric/tree/main 
-"""
-LEADERBOARD_MD['TACTL'] = """Threat Actor Competency Test for LLMs (TACTL) is a multiple-choice benchmark as a challenging offensive cyber knowledge test. 
-
-Paper: https://arxiv.org/abs/2502.15797
-Code: They plan to open-source TACTL (https://gbhackers.com/mitre-releases-occult-framework/). 
-"""
-LEADERBOARD_MD['AutoPenBench'] = """AutoPenBench is an open benchmark for evaluating generative agents in automated penetration testing.
-
-Paper: https://arxiv.org/abs/2410.03225 
-Code: https://github.com/lucagioacchini/auto-pen-bench 
-"""
-LEADERBOARD_MD['PrimeVul'] = """PrimeVul is a dataset for training and evaluating code LMs for vulnerability detection.
-
-Paper: https://arxiv.org/abs/2403.18624 
-Code: https://github.com/DLVulDet/PrimeVul
-"""
-LEADERBOARD_MD['CRUXEval'] = """CRUXEval (Code Reasoning, Understanding, and eXecution Evaluation) is a benchmark consisting of 800 Python functions (3-13 lines).
-
-Paper: https://arxiv.org/abs/2401.03065 
-Code: https://github.com/facebookresearch/cruxeval 
-"""
-LEADERBOARD_MD['SWE-bench-verified'] = """This is a human-validated subset of SWE-bench that more reliably evaluates AI models' ability to solve real-world software issues.
-
-Paper: https://openai.com/index/introducing-swe-bench-verified/ 
-Code: https://github.com/swe-bench/SWE-bench 
+VLMEVALKIT_README = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/README.md'
+# CONSTANTS-CITATION
+CITATION_BUTTON_TEXT = r"""@article{guo2025sok,
+  title={{Frontier AI's Impact on the Cybersecurity Landscape}},
+  author={Guo, Wenbo and Potter, Yujin and Shi, Tianneng and Wang, Zhun and Zhang, Andy and Song, Dawn},
+  journal={arXiv preprint arXiv:2504.05408},
+  year={2025}
+}
+"""
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+# CONSTANTS-TEXT
+LEADERBORAD_INTRODUCTION = """# Frontier AI Cybersecurity Observatory
+### Welcome to Frontier AI Cybersecurity Observatory! This leaderboard is a collection of benchmarks relevant to cybersecurity capabilities. 
+
+Tracking AI capabilities in cybersecurity is essential for understanding emerging impacts and risks. Our Frontier AI Cybersecurity Observatory provides a centralized platform that aggregates relevant benchmarks, enabling the community to more easily monitor and assess the evolving cybersecurity capabilities of AI systems.
+
+This leaderboard covers {} benchmarks.
+
+This leaderboard was last updated: {} """
+# CONSTANTS-FIELDS
+# META_FIELDS = [
+#     'Model'
+# ]
+
+DEFAULT_TASK = [
+    'Vulnerable Code Generation', 'Attack Generation', 'CTF', 'Cyber Knowledge', 'Pen Test', 'Vulnerability Detection', 'PoC Generation', 'Patching'
+]
+
+# The README file for each benchmark
+LEADERBOARD_MD = {}
+
+LEADERBOARD_MD['CyberSecEval-3'] = """CyberSecEval-3 is a security benchmarks for LLMs. CyberSecEval-3 assesses 8 different risks across two broad categories: risk to third parties, and risk to application developers and end users.  
+
+Paper: https://arxiv.org/abs/2408.01605 
+Code: https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks 
+"""
+LEADERBOARD_MD['SecCodePLT'] = """ SecCodePLT is a unified and comprehensive evaluation platform for code GenAIs' risks. This benchmark consists of insecure coding tasks and cyberattack helpfulness tasks. The helpfulness tasks are designed considering five attack steps: reconnaissance, weaponization & infiltration, C2 & execution, discovery, and collection.
+
+Paper: https://arxiv.org/abs/2410.11096 
+Code: https://github.com/CodeSecPLT/CodeSecPLT 
+"""
+LEADERBOARD_MD['RedCode'] = """RedCode is a benchmark for risky code execution and generation: (1) RedCode-Exec provides challenging prompts that could lead to risky code execution, aiming to evaluate code agents' ability to recognize and handle unsafe code. (2) RedCode-Gen provides 160 prompts with function signatures and docstrings as input to assess whether code agents will follow instructions to generate harmful code or software.
+
+Paper: https://arxiv.org/abs/2411.07781
+Code: https://github.com/AI-secure/RedCode 
+"""
+LEADERBOARD_MD['CyBench'] = """Cybench is a framework for specifying cybersecurity tasks and evaluating agents on those tasks. This includes 40 professional-level Capture the Flag (CTF) tasks from 4 distinct CTF competitions, chosen to be recent, meaningful, and spanning a wide range of difficulties.
+
+Paper: https://arxiv.org/abs/2408.08926 
+Code: https://github.com/andyzorigin/cybench 
+"""
+LEADERBOARD_MD['NYU CTF Bench'] = """This assesses LLMs in solving CTF challenges. This includes a diverse range of CTF challenges from popular competitions.
+
+Paper: https://arxiv.org/abs/2406.05590 
+Code: https://github.com/NYU-LLM-CTF/NYU_CTF_Bench 
+"""
+LEADERBOARD_MD['CyberBench'] = """CyberBench is a multi-task benchmark to evaluate the model knowledge in cybersecurity. 
+
+Paper: https://zefang-liu.github.io/files/liu2024cyberbench_paper.pdf 
+Code: https://github.com/jpmorganchase/CyberBench
+"""
+LEADERBOARD_MD['CyberMetric'] = """CyberMetric is designed to accurately test the general knowledge of LLMs in cybersecurity. CyberMetric-80, CyberMetric-500, CyberMetric-2000, and CyberMetric-10000 are multiple-choice Q&A benchmark datasets comprising 80, 500, 2000, and 10,000 questions, respectively.
+
+Paper: https://arxiv.org/abs/2402.07688 
+Code: https://github.com/cybermetric/CyberMetric/tree/main 
+"""
+LEADERBOARD_MD['TACTL'] = """Threat Actor Competency Test for LLMs (TACTL) is a multiple-choice benchmark as a challenging offensive cyber knowledge test. 
+
+Paper: https://arxiv.org/abs/2502.15797
+Code: They plan to open-source TACTL (https://gbhackers.com/mitre-releases-occult-framework/). 
+"""
+LEADERBOARD_MD['AutoPenBench'] = """AutoPenBench is an open benchmark for evaluating generative agents in automated penetration testing.
+
+Paper: https://arxiv.org/abs/2410.03225 
+Code: https://github.com/lucagioacchini/auto-pen-bench 
+"""
+LEADERBOARD_MD['PrimeVul'] = """PrimeVul is a dataset for training and evaluating code LMs for vulnerability detection.
+
+Paper: https://arxiv.org/abs/2403.18624 
+Code: https://github.com/DLVulDet/PrimeVul
+"""
+LEADERBOARD_MD['CRUXEval'] = """CRUXEval (Code Reasoning, Understanding, and eXecution Evaluation) is a benchmark consisting of 800 Python functions (3-13 lines).
+
+Paper: https://arxiv.org/abs/2401.03065 
+Code: https://github.com/facebookresearch/cruxeval 
+"""
+LEADERBOARD_MD['SWE-bench-verified'] = """This is a human-validated subset of SWE-bench that more reliably evaluates AI models' ability to solve real-world software issues.
+
+Paper: https://openai.com/index/introducing-swe-bench-verified/ 
+Code: https://github.com/swe-bench/SWE-bench 
 """