Apply snapshot download

app.py

@@ -7,7 +7,7 @@ from distutils.util import strtobool
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import HfApi
+from huggingface_hub import HfApi, snapshot_download
 
 from src.assets.css_html_js import custom_css, get_window_url_params
 from src.assets.text_content import (
@@ -28,7 +28,7 @@ from src.display_models.utils import (
     styled_message,
     styled_warning,
 )
-from src.load_from_hub import get_evaluation_queue_df, get_leaderboard_df, is_model_on_hub, load_all_info_from_hub
+from src.load_from_hub import get_all_requested_models, get_evaluation_queue_df, get_leaderboard_df, is_model_on_hub
 from src.rate_limiting import user_submission_permission
 
 pd.set_option("display.precision", 1)
@@ -86,22 +86,12 @@ BENCHMARK_COLS = [
     ]
 ]
 
-## LOAD INFO FROM HUB
-eval_queue, requested_models, eval_results, users_to_submission_dates = load_all_info_from_hub(
-    QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH
-)
-
-if not IS_PUBLIC:
-    (eval_queue_private, requested_models_private, eval_results_private, _) = load_all_info_from_hub(
-        PRIVATE_QUEUE_REPO,
-        PRIVATE_RESULTS_REPO,
-        EVAL_REQUESTS_PATH_PRIVATE,
-        EVAL_RESULTS_PATH_PRIVATE,
-    )
-else:
-    eval_queue_private, eval_results_private = None, None
+snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None)
+snapshot_download(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None)
+requested_models, users_to_submission_dates = get_all_requested_models(EVAL_REQUESTS_PATH)
 
-original_df = get_leaderboard_df(eval_results, eval_results_private, COLS, BENCHMARK_COLS)
+original_df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
+leaderboard_df = original_df.copy()
 models = original_df["model_name_for_query"].tolist() # needed for model backlinks in their to the leaderboard
 
 # Commented out because it causes infinite restart loops in local
@@ -112,13 +102,12 @@ models = original_df["model_name_for_query"].tolist() # needed for model backlin
 
 # print(to_be_dumped)
 
-leaderboard_df = original_df.copy()
 (
     finished_eval_queue_df,
     running_eval_queue_df,
     pending_eval_queue_df,
     failed_eval_queue_df,
-) = get_evaluation_queue_df(eval_queue, eval_queue_private, EVAL_REQUESTS_PATH, EVAL_COLS)
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 ## INTERACTION FUNCTIONS
 def add_new_eval(
@@ -157,6 +146,27 @@ def add_new_eval(
         model_on_hub, error = is_model_on_hub(model, revision)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')
+    
+    model_info = api.model_info(repo_id=model, revision=revision)
+
+    size_pattern = re.compile(r"(\d+\.)?\d+(b|m)")
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except AttributeError:
+        try:
+            size_match = re.search(size_pattern, model.lower())
+            model_size = size_match.group(0)
+            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
+        except AttributeError:
+            return 65
+
+    size_factor = 8 if (precision == "GPTQ" or "GPTQ" in model) else 1
+    model_size = size_factor * model_size
+
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        license = "?"
 
     print("adding new eval")
 

model_info_cache.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df89e55d44e9203e4902e734bdf72fd21315a9e2e9dbee43fffb04c199019f59
-size 595504
+oid sha256:337f1fb80e92327e7c7b130c03617439f7923e3f7c5383f5abb07e017ef9cae3
+size 715983

model_size_cache.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df80f59fb639788c197e0b7ca13de55c24cc2ae54ca9c67f54ef730515f7032b
-size 16828
+oid sha256:64d63b51e6f5d6dd985b44ef6ddf513d9a7a138e734d77ae7382fd7a49a137ea
+size 20652

src/display_models/read_results.py

@@ -113,10 +113,10 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
     return result_key, eval_results
 
 
-def get_eval_results() -> List[EvalResult]:
+def get_eval_results(results_path: str) -> List[EvalResult]:
     json_filepaths = []
 
-    for root, dir, files in os.walk("eval-results" + ("-private" if not IS_PUBLIC else "")):
+    for root, dir, files in os.walk(results_path + ("-private" if not IS_PUBLIC else "")):
         # We should only have json files in model results
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
@@ -146,7 +146,7 @@ def get_eval_results() -> List[EvalResult]:
     return eval_results
 
 
-def get_eval_results_dicts() -> List[Dict]:
-    eval_results = get_eval_results()
+def get_eval_results_dicts(results_path: str) -> List[Dict]:
+    eval_results = get_eval_results(results_path)
 
     return [e.to_dict() for e in eval_results]

src/load_from_hub.py

@@ -1,10 +1,9 @@
 import json
 import os
+from collections import defaultdict
 
 import pandas as pd
-from huggingface_hub import Repository
 from transformers import AutoConfig
-from collections import defaultdict
 
 from src.assets.hardcoded_evals import baseline
 from src.display_models.get_model_metadata import apply_metadata
@@ -35,43 +34,8 @@ def get_all_requested_models(requested_models_dir: str) -> set[str]:
     return set(file_names), users_to_submission_dates
 
 
-def load_all_info_from_hub(QUEUE_REPO: str, RESULTS_REPO: str, QUEUE_PATH: str, RESULTS_PATH: str) -> list[Repository]:
-    eval_queue_repo = None
-    eval_results_repo = None
-    requested_models = None
-
-    print("Pulling evaluation requests and results.")
-
-    eval_queue_repo = Repository(
-        local_dir=QUEUE_PATH,
-        clone_from=QUEUE_REPO,
-        repo_type="dataset",
-    )
-    eval_queue_repo.git_pull()
-
-    eval_results_repo = Repository(
-        local_dir=RESULTS_PATH,
-        clone_from=RESULTS_REPO,
-        repo_type="dataset",
-    )
-    eval_results_repo.git_pull()
-
-    requested_models, users_to_submission_dates = get_all_requested_models("eval-queue")
-
-    return eval_queue_repo, requested_models, eval_results_repo, users_to_submission_dates
-
-
-def get_leaderboard_df(
-    eval_results: Repository, eval_results_private: Repository, cols: list, benchmark_cols: list
-) -> pd.DataFrame:
-    if eval_results:
-        print("Pulling evaluation results for the leaderboard.")
-        eval_results.git_pull()
-    if eval_results_private:
-        print("Pulling evaluation results for the leaderboard.")
-        eval_results_private.git_pull()
-
-    all_data = get_eval_results_dicts()
+def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    all_data = get_eval_results_dicts(results_path)
 
     # all_data.append(baseline)
     apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
@@ -85,15 +49,7 @@ def get_leaderboard_df(
     return df
 
 
-def get_evaluation_queue_df(
-    eval_queue: Repository, eval_queue_private: Repository, save_path: str, cols: list
-) -> list[pd.DataFrame]:
-    if eval_queue:
-        print("Pulling changes for the evaluation queue.")
-        eval_queue.git_pull()
-    if eval_queue_private:
-        print("Pulling changes for the evaluation queue.")
-        eval_queue_private.git_pull()
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
 
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
@@ -143,6 +99,5 @@ def is_model_on_hub(model_name: str, revision: str) -> bool:
             "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
         )
 
-    except Exception as e:
-        print(f"Could not get the model config from the hub.: {e}")
+    except Exception:
         return False, "was not found on hub!"