add auto eval

app.py

@@ -24,7 +24,7 @@ from src.display.utils import (
     WeightType,
     Precision
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.envs import API, EVAL_REQUESTS_PATH, AUTO_RESULTS_PATH, HUMAN_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 
@@ -41,7 +41,8 @@ try:
 except Exception:
     restart_space()
 try:
-    print(EVAL_RESULTS_PATH)
+    print(AUTO_RESULTS_PATH)
+    print(HUMAN_RESULTS_PATH)
     # snapshot_download(
     #     repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
     # )
@@ -49,7 +50,8 @@ except Exception:
     restart_space()
 
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+AUTO_LEADERBOARD_DF = get_leaderboard_df(AUTO_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+HUMAN_LEADERBOARD_DF = get_leaderboard_df(HUMAN_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 
 # (
 #     finished_eval_queue_df,
@@ -97,8 +99,11 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+        with gr.TabItem("🏆OmniEval-Human", elem_id="llm-benchmark-tab-table", id=0):
+            leaderboard = init_leaderboard(HUMAN_LEADERBOARD_DF)
+
+        with gr.TabItem("🤖OmniEval-Auto", elem_id="llm-benchmark-tab-table", id=1):
+            leaderboard = init_leaderboard(AUTO_LEADERBOARD_DF)
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

eval-results/omnieval-auto/CLOSE_deepseek-v2-chat/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,34 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.0,
+      "map": 0.0
+    },
+    "generation": {
+      "em": 0.0011680767773708802,
+      "f1": 0.3709233008524321,
+      "rouge1": 0.2570830224992733,
+      "rouge2": 0.09085043984411759,
+      "rougeL": 0.1860727124152372,
+      "accuracy": 0.35869427958075517,
+      "completeness": 0.5755086661642803,
+      "hallucination": 0.0,
+      "utilization": 0.0,
+      "numerical_accuracy": 0.11213720316622691
+    }
+  },
+  "config": {
+    "eval_name": "CLOSE_deepseek-v2-chat",
+    "generative_model": "deepseek-ai/DeepSeek-V2-Chat-0628",
+    "generative_model_args": {
+      "name": "deepseek-ai/DeepSeek-V2-Chat-0628",
+      "num_params": 236,
+      "open_source": true
+    },
+    "retrieval_model": "CLOSE",
+    "retrieval_model_args": {
+      "num_params": 0.0,
+      "open_source": true
+    }
+  }
+}

eval-results/omnieval-auto/CLOSE_llama3-70b-instruct/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,34 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.0,
+      "map": 0.0
+    },
+    "generation": {
+      "em": 0.0008839499936860714,
+      "f1": 0.39891051266403244,
+      "rouge1": 0.2679937299203498,
+      "rouge2": 0.09293819886242284,
+      "rougeL": 0.19931718897529843,
+      "accuracy": 0.3238413941154186,
+      "completeness": 0.52843637454982,
+      "hallucination": 0.0,
+      "utilization": 0.0,
+      "numerical_accuracy": 0.06765619606489472
+    }
+  },
+  "config": {
+    "eval_name": "CLOSE_llama3-70b-instruct",
+    "generative_model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+    "generative_model_args": {
+      "name": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+      "num_params": 70.6,
+      "open_source": true
+    },
+    "retrieval_model": "CLOSE",
+    "retrieval_model_args": {
+      "num_params": 0.0,
+      "open_source": true
+    }
+  }
+}

eval-results/omnieval-auto/CLOSE_qwen2-72b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,34 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.0,
+      "map": 0.0
+    },
+    "generation": {
+      "em": 0.0002525571410531633,
+      "f1": 0.32215271896313463,
+      "rouge1": 0.2352109086389165,
+      "rouge2": 0.08060449522198783,
+      "rougeL": 0.16073680618083347,
+      "accuracy": 0.37883571157974494,
+      "completeness": 0.6016923768159353,
+      "hallucination": 0.0,
+      "utilization": 0.0,
+      "numerical_accuracy": 0.1255931667193926
+    }
+  },
+  "config": {
+    "eval_name": "CLOSE_qwen2-72b",
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72.7,
+      "open_source": true
+    },
+    "retrieval_model": "CLOSE",
+    "retrieval_model_args": {
+      "num_params": 0.0,
+      "open_source": true
+    }
+  }
+}

eval-results/omnieval-auto/CLOSE_yi15-34b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,34 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.0,
+      "map": 0.0
+    },
+    "generation": {
+      "em": 0.0,
+      "f1": 0.06725057117657031,
+      "rouge1": 0.1277764944666756,
+      "rouge2": 0.03211441875898112,
+      "rougeL": 0.03257144660565082,
+      "accuracy": 0.15734309887612072,
+      "completeness": 0.5063249001331558,
+      "hallucination": 0.0,
+      "utilization": 0.0,
+      "numerical_accuracy": 0.06932865291794647
+    }
+  },
+  "config": {
+    "eval_name": "CLOSE_yi15-34b",
+    "generative_model": "01ai/Yi-1.5-34B-Chat-16K",
+    "generative_model_args": {
+      "name": "01ai/Yi-1.5-34B-Chat-16K",
+      "num_params": 34.4,
+      "open_source": true
+    },
+    "retrieval_model": "CLOSE",
+    "retrieval_model_args": {
+      "num_params": 0.0,
+      "open_source": true
+    }
+  }
+}

eval-results/omnieval-auto/bge-large-zh_qwen2-72b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.3097634381445468,
+      "map": 0.30402197247127166
+    },
+    "generation": {
+      "em": 0.0026518499810582142,
+      "f1": 0.2480828824153542,
+      "rouge1": 0.2493538725800514,
+      "rouge2": 0.1235656068292625,
+      "rougeL": 0.16098924930699862,
+      "accuracy": 0.3906427579239803,
+      "completeness": 0.5930474914396308,
+      "hallucination": 0.0,
+      "utilization": 0.5045650189122212,
+      "numerical_accuracy": 0.28149656401119877
+    }
+  },
+  "config": {
+    "eval_name": "bge-large-zh_qwen2-72b",
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72.7,
+      "open_source": true
+    },
+    "retrieval_model": "BAAI/bge-large-zh",
+    "retrieval_model_args": {
+      "name": "BAAI/bge-large-zh",
+      "num_params": 0.326,
+      "open_source": true
+    }
+  }
+}

eval-results/omnieval-auto/bge-m3_qwen2-72b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.33076566906595944,
+      "map": 0.32402765500694536
+    },
+    "generation": {
+      "em": 0.002525571410531633,
+      "f1": 0.2524796046548042,
+      "rouge1": 0.2542055585319881,
+      "rouge2": 0.12967013110722864,
+      "rougeL": 0.16623387811734364,
+      "accuracy": 0.0,
+      "completeness": 0.0,
+      "hallucination": 0.0,
+      "utilization": 0.0,
+      "numerical_accuracy": 0.0
+    }
+  },
+  "config": {
+    "eval_name": "bge-m3_qwen2-72b",
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72.7,
+      "open_source": true
+    },
+    "retrieval_model": "BAAI/bge-m3",
+    "retrieval_model_args": {
+      "name": "BAAI/bge-m3",
+      "num_params": 0.5,
+      "open_source": true
+    }
+  }
+}

eval-results/omnieval-auto/e5-mistral-7b_qwen2-72b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.26059266742433806,
+      "map": 0.25533526960474806
+    },
+    "generation": {
+      "em": 0.002146735698951888,
+      "f1": 0.24207930410773865,
+      "rouge1": 0.24073805243800728,
+      "rouge2": 0.1162276261848681,
+      "rougeL": 0.1534679545927458,
+      "accuracy": 0.37713095087763604,
+      "completeness": 0.5855007473841555,
+      "hallucination": 0.0,
+      "utilization": 0.49136152656008253,
+      "numerical_accuracy": 0.2582123758594347
+    }
+  },
+  "config": {
+    "eval_name": "e5-mistral-7b_qwen2-72b",
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72.7,
+      "open_source": true
+    },
+    "retrieval_model": "intfloat/e5-mistral-7b-instruct",
+    "retrieval_model_args": {
+      "name": "intfloat/e5-mistral-7b-instruct",
+      "num_params": 7.11,
+      "open_source": true
+    }
+  }
+}

eval-results/omnieval-auto/gte-qwen2-1.5b_deepseek-v2-chat/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.3406848507808225,
+      "map": 0.3337426863661236
+    },
+    "generation": {
+      "em": 0.0035568464031653824,
+      "f1": 0.3226028700822056,
+      "rouge1": 0.29804464952499493,
+      "rouge2": 0.1619392409911174,
+      "rougeL": 0.21536150159516076,
+      "accuracy": 0.3783377209477247,
+      "completeness": 0.5935541629364369,
+      "hallucination": 0.06668379802132854,
+      "utilization": 0.48314821907315203,
+      "numerical_accuracy": 0.2761605035405193
+    }
+  },
+  "config": {
+    "eval_name": "gte-qwen2-1.5b_deepseek-v2-chat",
+    "generative_model": "deepseek-ai/DeepSeek-V2-Chat-0628",
+    "generative_model_args": {
+      "name": "deepseek-ai/DeepSeek-V2-Chat-0628",
+      "num_params": 236,
+      "open_source": true
+    },
+    "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+    "retrieval_model_args": {
+      "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+      "num_params": 1.78,
+      "open_source": true
+    }
+  }
+}

eval-results/omnieval-auto/gte-qwen2-1.5b_llama3-70b-instruct/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.3406848507808225,
+      "map": 0.3337426863661236
+    },
+    "generation": {
+      "em": 0.030906680136380857,
+      "f1": 0.4704248712273675,
+      "rouge1": 0.3844331865430577,
+      "rouge2": 0.21544656691735142,
+      "rougeL": 0.3082188596657867,
+      "accuracy": 0.4181714862987751,
+      "completeness": 0.586105675146771,
+      "hallucination": 0.0880543450397334,
+      "utilization": 0.45601078859491395,
+      "numerical_accuracy": 0.2751721876024926
+    }
+  },
+  "config": {
+    "eval_name": "gte-qwen2-1.5b_llama3-70b-instruct",
+    "generative_model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+    "generative_model_args": {
+      "name": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+      "num_params": 70.6,
+      "open_source": true
+    },
+    "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+    "retrieval_model_args": {
+      "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+      "num_params": 1.78,
+      "open_source": true
+    }
+  }
+}

eval-results/{demo-leaderboard → omnieval-auto}/gte-qwen2-1.5b_qwen2-72b/results_2023-12-08 15:46:20.425378.json

@@ -1,20 +1,20 @@
 {
   "results": {
     "retrieval": {
-      "mrr": 0.36173120728929387,
-      "map": 0.3512338648443432
+      "mrr": 0.3406848507808225,
+      "map": 0.3337426863661236
     },
     "generation": {
-      "em": 0.002277904328018223,
-      "f1": 0.3804001391052641,
-      "rouge1": 0.34576336184459094,
-      "rouge2": 0.1928778762677512,
-      "rougeL": 0.2383694455084706,
-      "accuracy": 0.4145785876993166,
-      "completeness": 0.598297213622291,
-      "hallucination": 0.07213496218731821,
-      "utilization": 1.13922942206655,
-      "numerical_accuracy": 0.3218694885361552
+      "em": 0.0028412678368480867,
+      "f1": 0.2477112059712835,
+      "rouge1": 0.25666135328401396,
+      "rouge2": 0.13256084364546591,
+      "rougeL": 0.1669344569228441,
+      "accuracy": 0.40573304710190683,
+      "completeness": 0.6131668895824045,
+      "hallucination": 0.0,
+      "utilization": 0.5346272891410885,
+      "numerical_accuracy": 0.2971301335972291
     }
   },
   "config": {

eval-results/omnieval-auto/gte-qwen2-1.5b_yi15-34b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.3406848507808225,
+      "map": 0.3337426863661236
+    },
+    "generation": {
+      "em": 0.0,
+      "f1": 0.09732568803130702,
+      "rouge1": 0.1642342072893325,
+      "rouge2": 0.06542075931397044,
+      "rougeL": 0.059256539829821125,
+      "accuracy": 0.3304375804375804,
+      "completeness": 0.5735068912710567,
+      "hallucination": 0.06555017663221248,
+      "utilization": 0.4132755170113409,
+      "numerical_accuracy": 0.175
+    }
+  },
+  "config": {
+    "eval_name": "gte-qwen2-1.5b_yi15-34b",
+    "generative_model": "01ai/Yi-1.5-34B-Chat-16K",
+    "generative_model_args": {
+      "name": "01ai/Yi-1.5-34B-Chat-16K",
+      "num_params": 34.4,
+      "open_source": true
+    },
+    "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+    "retrieval_model_args": {
+      "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+      "num_params": 1.78,
+      "open_source": true
+    }
+  }
+}

eval-results/omnieval-auto/jina-zh_qwen2-72b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.25315906890600665,
+      "map": 0.24830681483352277
+    },
+    "generation": {
+      "em": 0.0026518499810582142,
+      "f1": 0.24837825152624493,
+      "rouge1": 0.24111819423215256,
+      "rouge2": 0.11665848753826197,
+      "rougeL": 0.1558018779014647,
+      "accuracy": 0.3705644652102538,
+      "completeness": 0.5820335932813437,
+      "hallucination": 0.0,
+      "utilization": 0.4738984364905027,
+      "numerical_accuracy": 0.24648820567187915
+    }
+  },
+  "config": {
+    "eval_name": "jina-zh_qwen2-72b",
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72.7,
+      "open_source": true
+    },
+    "retrieval_model": "jinaai/jina-embeddings-v2-base-zh",
+    "retrieval_model_args": {
+      "name": "jinaai/jina-embeddings-v2-base-zh",
+      "num_params": 0.161,
+      "open_source": true
+    }
+  }
+}

eval-results/{demo-leaderboard/qwen2-72b_bge-large-zh → omnieval-human/bge-large-zh_qwen2-72b}/results_2023-12-08 15:46:20.425378.json

@@ -18,7 +18,7 @@
     }
   },
   "config": {
-    "eval_name": "qwen2-72b_bge-large-zh",
+    "eval_name": "bge-large-zh_qwen2-72b",
     "generative_model": "Qwen/Qwen2.5-72B-Instruct",
     "generative_model_args": {
       "name": "Qwen/Qwen2.5-72B-Instruct",

eval-results/{demo-leaderboard/qwen2-72b_bge-m3 → omnieval-human/bge-m3_qwen2-72b}/results_2023-12-08 15:46:20.425378.json

@@ -18,7 +18,7 @@
     }
   },
   "config": {
-    "eval_name": "qwen2-72b_bge-m3",
+    "eval_name": "bge-m3_qwen2-72b",
     "generative_model": "Qwen/Qwen2.5-72B-Instruct",
     "generative_model_args": {
       "name": "Qwen/Qwen2.5-72B-Instruct",

eval-results/{demo-leaderboard/qwen2-72b_e5-mistral-7b → omnieval-human/e5-mistral-7b_qwen2-72b}/results_2023-12-08 15:46:20.425378.json

@@ -18,7 +18,7 @@
     }
   },
   "config": {
-    "eval_name": "qwen2-72b_e5-mistral-7b",
+    "eval_name": "e5-mistral-7b_qwen2-72b",
     "generative_model": "Qwen/Qwen2.5-72B-Instruct",
     "generative_model_args": {
       "name": "Qwen/Qwen2.5-72B-Instruct",

eval-results/{demo-leaderboard/qwen2-72b_gte-qwen2-1.5b → omnieval-human/gte-qwen2-1.5b_qwen2-72b}/results_2023-12-08 15:46:20.425378.json

@@ -18,7 +18,7 @@
     }
   },
   "config": {
-    "eval_name": "qwen2-72b_gte-qwen2-1.5b",
+    "eval_name": "gte-qwen2-1.5b_qwen2-72b",
     "generative_model": "Qwen/Qwen2.5-72B-Instruct",
     "generative_model_args": {
       "name": "Qwen/Qwen2.5-72B-Instruct",

eval-results/{demo-leaderboard/qwen2-72b_jina-zh → omnieval-human/jina-zh_qwen2-72b}/results_2023-12-08 15:46:20.425378.json

@@ -18,7 +18,7 @@
     }
   },
   "config": {
-    "eval_name": "qwen2-72b_jina-zh",
+    "eval_name": "jina-zh_qwen2-72b",
     "generative_model": "Qwen/Qwen2.5-72B-Instruct",
     "generative_model_args": {
       "name": "Qwen/Qwen2.5-72B-Instruct",

src/about.py

@@ -83,12 +83,12 @@ LLM_BENCHMARKS_TEXT = f"""
 With FlashRAG and provided resources, you can effortlessly reproduce existing SOTA works in the RAG domain or implement your custom RAG processes and components. -->
 
 
-## :wrench: Installation
+## 🔧 Installation
 `conda env create -f environment.yml && conda activate finrag`
 
-<!-- ## :sparkles: Features
+<!-- ## ✨ Features
 1. -->
-## :rocket: Quick-Start
+## 🚀 Quick-Start
 Notion:
 1. The code run path is `./OpenFinBench`
 2. We provide our auto-generated evaluation dataset in <a href="https://huggingface.co/datasets/RUC-NLPIR/FlashRAG_datasets/" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Dataset-27b3b4></a>
@@ -136,11 +136,11 @@ Then conduct the model-based evaluate using the following codes, (change the par
 sh evaluator/judgement/judger.sh 
 ``` 
 
-## :bookmark: License
+## 🔖 License
 
 OmniEval is licensed under the [<u>MIT License</u>](./LICENSE).
 
-## :star2: Citation
+## 🌟 Citation
 The paper is waiting to be released!
 
 <!-- # Check Infos

src/envs.py

@@ -6,7 +6,7 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "RUC-NLPIR" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"
@@ -18,7 +18,8 @@ CACHE_PATH=os.getenv("HF_HOME", ".")
 
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+HUMAN_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results", "omnieval-human")
+AUTO_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results", "omnieval-auto")
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 

src/leaderboard/read_evals.py

@@ -183,7 +183,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
-
+    print(f"Reading results from {results_path}")
     for root, _, files in os.walk(results_path):
         # We should only have json files in model results
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):