new build

README.md

@@ -1,14 +1,33 @@
 ---
 title: TransformerRanker
-emoji: 🎯
+emoji: ⚡️
 colorFrom: yellow
-colorTo: purple
+colorTo: indigo
 sdk: gradio
 sdk_version: 5.44.0
-app_file: demo/app.py
-pinned: false
+app_file: launch.py
+pinned: true
 license: mit
 short_description: Efficient LM Ranking for Downstream Tasks
---- 
+tags:
+  - ranking
+  - research
+  - evaluation
+  - llm
+  - demo
+---
+
+## Run Demo
+
+Install in editable mode:
+
+``` bash
+pip install -e .
+```
+
+Launch it:
+
+``` bash
+python launch.py
+```
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

demo/app.py

@@ -2,11 +2,17 @@ import gradio as gr
 from datasets import disable_caching, load_dataset
 from transformer_ranker import TransformerRanker
 
-from demo.config import SAMPLE_SIZE, MAX_SAMPLE_SIZE, ALL_LMS, PRESELECTED_LMS, GRADIO_THEME
+from demo.config import SAMPLE_SIZE, MAX_SAMPLE_SIZE, ALL_LMS, PRESELECTED_LMS
 from demo.utils import (
-    BANNER, FOOTER, CSS, UNSET,
-    EmbeddingProgressTracker, compute_ratio,
-    validate_dataset, preprocess_dataset, ensure_dataset_is_loaded
+    BANNER,
+    FOOTER,
+    CSS,
+    UNSET,
+    EmbeddingProgressTracker,
+    compute_ratio,
+    validate_dataset,
+    preprocess_dataset,
+    ensure_dataset_is_loaded,
 )
 
 
@@ -18,31 +24,33 @@ with gr.Blocks(css=CSS, theme=None) as demo:
 
     ##### 1. Load from datasets #####
 
-    gr.Markdown("## Load Downstream Dataset")
+    gr.Markdown("## 📚 Load Data")
 
     gr.Markdown(
-        "Select a dataset from the Hugging Face Hub such as `trec`. "
-        "This defines your downstream task."
+        "Pick a dataset from the Hugging Face Hub (e.g. `trec`). This defines your downstream task."
     )
 
     with gr.Group():
         dataset = gr.State(None)
 
         dataset_id = gr.Textbox(
-            label="Dataset name",
+            label="Dataset identifier",
             placeholder="try: trec, conll2003, ag_news",
             max_lines=1,
         )
 
-        load_dataset_button = gr.Button(value="Load data", variant="primary", interactive=True,)
+        load_dataset_button = gr.Button(
+            value="Load data",
+            variant="primary",
+            interactive=True,
+        )
 
         # enable loading if dataset exists on hub
         dataset_id.change(validate_dataset, inputs=dataset_id, outputs=load_dataset_button)
 
     gr.Markdown(
-        "Settings auto-configured. "
-        "Adjust the downsampling ratio in Dataset Setup, "
-        "or use the complete dataset with the [framework](https://github.com/flairNLP/transformer-ranker)."
+        "⚡️ Speed mode on: tweak the downsampling ratio in *Dataset Setup* for quicker runs. "
+        "Unlock the full data via [framework](https://github.com/flairNLP/transformer-ranker)."
     )
 
     ##### data preprocessing #####
@@ -52,9 +60,7 @@ with gr.Blocks(css=CSS, theme=None) as demo:
             dataset_id_label = gr.Label("", label="Dataset")
             num_samples = gr.State(0)
             num_samples_label = gr.Label("", label="Dataset size")
-            num_samples.change(
-                lambda x: str(x), inputs=[num_samples], outputs=[num_samples_label]
-            )
+            num_samples.change(lambda x: str(x), inputs=[num_samples], outputs=[num_samples_label])
 
         with gr.Row():
             text_column = gr.Dropdown("", label="Text Column")
@@ -66,9 +72,7 @@ with gr.Blocks(css=CSS, theme=None) as demo:
 
         with gr.Group():
             downsample_ratio = gr.State(0.0)
-            sampling_rate = gr.Slider(
-                20, MAX_SAMPLE_SIZE, label="Sampling rate", value=SAMPLE_SIZE, step=1
-            )
+            sampling_rate = gr.Slider(20, MAX_SAMPLE_SIZE, label="Sampling rate", value=SAMPLE_SIZE, step=1)
             downsample_ratio_label = gr.Label("", label="Sampling rate")
             downsample_ratio.change(
                 lambda x: f"{x:.1%}",
@@ -87,20 +91,14 @@ with gr.Blocks(css=CSS, theme=None) as demo:
                 outputs=downsample_ratio,
             )
 
-    # load and show details
     def load_hf_dataset(dataset_id):
         try:
             dataset = load_dataset(dataset_id, trust_remote_code=True)
             dataset_details = preprocess_dataset(dataset)
         except ValueError as e:
-            gr.Warning("Collections not supported. Load one dataset only.")
+            gr.Warning(f"Watch out — single datasets only. Cannot load dataset: {e}")
 
-        return (
-            gr.update(value="Loaded"),
-            dataset_id,
-            dataset,
-            *dataset_details
-        )
+        return (gr.update(value="Loaded"), dataset_id, dataset, *dataset_details)
 
     load_dataset_button.click(
         load_hf_dataset,
@@ -120,29 +118,24 @@ with gr.Blocks(css=CSS, theme=None) as demo:
 
     ########## 2. Select LMs ##########
 
-    gr.Markdown("## Select Language Models")
+    gr.Markdown("## 🧠 Select Language Models")
 
     gr.Markdown(
-        "Add two or more pretrained models for ranking. "
-        "Go with small models since this demo runs on CPU."
+        "Add two or more pretrained models to compare. "
+        "Stick to smaller models here since the demo runs on CPU."
     )
 
     with gr.Group():
-        model_options = [
-            (model_handle.split("/")[-1], model_handle)
-            for model_handle in ALL_LMS
-        ]
-        models = gr.CheckboxGroup(
-            choices=model_options, label="Model List", value=PRESELECTED_LMS
-        )
+        model_options = [(model_handle.split("/")[-1], model_handle) for model_handle in ALL_LMS]
+        models = gr.CheckboxGroup(choices=model_options, label="Model List", value=PRESELECTED_LMS)
 
     ########## 3. Run ranking ##########
 
-    gr.Markdown("## Rank Language Models")
+    gr.Markdown("## 🏆 Rank Models")
 
     gr.Markdown(
-        "Rank models by transferability to your downstream task. "
-        "Adjust the metric and layer aggregation in Advanced Settings."
+        "Rank models by transferability to your task. "
+        "More control? Tweak transferability metric and layer aggregation in *Settings*."
     )
 
     with gr.Group():
@@ -195,15 +188,13 @@ with gr.Blocks(css=CSS, theme=None) as demo:
     ):
 
         if text_column == UNSET:
-            raise gr.Error("Text column is not set.")
+            raise gr.Error("Text column is required.")
 
         if label_column == UNSET:
-            raise gr.Error("Label column is not set.")
+            raise gr.Error("Label column is required.")
 
         if task_category == UNSET:
-            raise gr.Error(
-                "Task category not set. Dataset must support classification or regression."
-            )
+            raise gr.Error("Task category is required.")
 
         if text_pair_column == UNSET:
             text_pair_column = None
@@ -229,23 +220,19 @@ with gr.Blocks(css=CSS, theme=None) as demo:
                     tracker=tracker,
                 )
 
-                sorted_results = sorted(
-                    results._results.items(), key=lambda item: item[1], reverse=True
-                )
-                return [
-                    (i + 1, model, score) for i, (model, score) in enumerate(sorted_results)
-                ]
+                sorted_results = sorted(results._results.items(), key=lambda item: item[1], reverse=True)
+                return [(i + 1, model, score) for i, (model, score) in enumerate(sorted_results)]
             except Exception as e:
-                print(e)
                 gr.Warning(f"Ranking issue: {e}")
                 return []
 
-    gr.Markdown("Ranking table → higher scores indicate better downstream performance.")
+    gr.Markdown("**Leaderboard:** higher score → better downstream performance.")
 
     ranking_results = gr.Dataframe(
         headers=["Rank", "Model", "Score"],
         datatype=["number", "str", "number"],
-        value=[["-", "-", "-"]]
+        value=[["-", "-", "-"]],
+        interactive=False
     )
 
     submit_button.click(
@@ -266,11 +253,3 @@ with gr.Blocks(css=CSS, theme=None) as demo:
     )
 
     gr.Markdown(FOOTER)
-
-if __name__ == "__main__":
-
-    # run up to 3 requests at once
-    demo.queue(default_concurrency_limit=3)
-
-    # run with 6 workers
-    demo.launch(max_threads=6)

demo/utils.py

@@ -9,11 +9,11 @@ from transformer_ranker.datacleaner import DatasetCleaner, TaskCategory
 from transformer_ranker.embedder import Embedder
 
 BANNER = """
-<h1 align="center">🔥 TransformerRanker 🔥</h1>
+<h1 align="center">🚀 TransformerRanker ⚡️</h1>
 
 <p align="center" style="max-width: 560px; margin: auto;">
     Find the best language model for your downstream task.
-    Load a dataset, select models from the 🤗 Hub, and rank them by <strong>transferability</strong>.
+    Load a dataset, pick models from the 🤗 Hub, and rank them by <strong>transferability</strong>.
 </p>
 
 <p align="center" style="font-weight: bold; margin-top: 20px; display: flex; justify-content: center; gap: 10px;">
@@ -35,8 +35,9 @@ BANNER = """
 """
 
 FOOTER = """
-**Note:** CPU-only quick demo. **Built by:** @lukasgarbas & @plonerma
-**Questions?** Open a [GitHub issue](https://github.com/flairNLP/transformer-ranker/issues) 🔫.
+**Note:** Quick CPU-only demo.   
+**Built by** [@lukasgarbas](https://huggingface.co/lukasgarbas) & [@plonerma](https://huggingface.co/plonerma)   
+**Questions?** Open a [GitHub issue](https://github.com/flairNLP/transformer-ranker/issues) 🔫
 """
 
 CSS = """
@@ -53,18 +54,18 @@ preprocessing = DatasetCleaner()
 
 
 def validate_dataset(dataset_name):
-    """Enable if dataset exists on Hub."""
+    """Quick look dataset existence on Hub."""
     try:
-        hf_api.dataset_info(dataset_name)  # quick dataset info call
+        hf_api.dataset_info(dataset_name)
         return gr.update(interactive=True)
-
     except (HTTPError, HFValidationError):
         return gr.update(value="Load data", interactive=False)
 
 
 def preprocess_dataset(dataset):
-    """Use data preprocessing to find text/label columns and task category."""
+    """Detect text/label columns and task type."""
     data = concatenate_datasets(list(dataset.values()))
+    sample_size = len(data)
 
     try:
         text_column = preprocessing._find_column(data, "text column")
@@ -83,19 +84,11 @@ def preprocess_dataset(dataset):
         try:
             task_category = preprocessing._find_task_category(data, label_column)
         except ValueError:
-            gr.Warning("Task category not auto-detected — framework supports classification, regression.")
-
-    text_column = gr.update(value=text_column, choices=data.column_names, interactive=True)
-    label_column = gr.update(value=label_column, choices=data.column_names, interactive=True)
-    text_pair  = gr.update(value=UNSET, choices=[UNSET, *data.column_names], interactive=True)
-    task_category = gr.update(value=task_category, choices=[str(t) for t in TaskCategory], interactive=True)
-    sample_size = len(data)
-
-    return task_category, text_column, text_pair, label_column, sample_size
+            gr.Warning(
+                "Task category not auto-detected — framework supports classification, regression."
+            )
 
-"""
     return (
-        text_column,
         gr.update(
             value=task_category,
             choices=[str(t) for t in TaskCategory],
@@ -110,9 +103,9 @@ def preprocess_dataset(dataset):
         gr.update(
             value=label_column, choices=data.column_names, interactive=True
         ),
-        num_samples,
+        sample_size,
     )
-"""
+
 
 def compute_ratio(num_samples_to_use, num_samples):
     if num_samples > 0:
@@ -128,12 +121,6 @@ def ensure_dataset_is_loaded(dataset, text_column, label_column, task_category):
         return gr.update(interactive=False)
 
 
-def ensure_one_lm_selected(checkbox_values, previous_values):
-    if not any(checkbox_values):
-        return previous_values
-    return checkbox_values
-
-
 # apply monkey patch to enable callbacks
 _old_embed = Embedder.embed
 
@@ -185,8 +172,6 @@ class EmbeddingProgressTracker:
             self.progress_bar(1.0, desc="Done")
         else:
             self.progress_bar(1.0, desc="Error")
-
-        # Do not suppress any errors
         return False
 
     def update_num_batches(self, total):
@@ -201,7 +186,6 @@ class EmbeddingProgressTracker:
 
     def update_bar(self):
         i = self.current_model
-
         description = f"Running {self.model_names[i]} ({i + 1} / {self.total})"
 
         progress = i / self.total

requirements.txt

@@ -1,5 +0,0 @@
-gradio>=5.0
-transformer-ranker==0.1.2
-transformers==4.41.0
-datasets==3.6
-protobuf

runtime.txt

@@ -1 +0,0 @@
-python-3.12