Spaces:
Building
Building
Commit
·
483e3a1
1
Parent(s):
e89d633
added example config
Browse files- app.py +2 -0
- src/assets/text_content.py +46 -1
app.py
CHANGED
|
@@ -9,6 +9,7 @@ from src.assets.text_content import (
|
|
| 9 |
TITLE,
|
| 10 |
INTRODUCTION_TEXT,
|
| 11 |
ABOUT_TEXT,
|
|
|
|
| 12 |
CITATION_BUTTON_LABEL,
|
| 13 |
CITATION_BUTTON_TEXT,
|
| 14 |
)
|
|
@@ -335,6 +336,7 @@ with demo:
|
|
| 335 |
|
| 336 |
with gr.TabItem("About 📖", id=3):
|
| 337 |
gr.HTML(ABOUT_TEXT, elem_classes="descriptive-text")
|
|
|
|
| 338 |
|
| 339 |
demo.load(
|
| 340 |
change_tab,
|
|
|
|
| 9 |
TITLE,
|
| 10 |
INTRODUCTION_TEXT,
|
| 11 |
ABOUT_TEXT,
|
| 12 |
+
EXAMPLE_CONFIG_TEXT,
|
| 13 |
CITATION_BUTTON_LABEL,
|
| 14 |
CITATION_BUTTON_TEXT,
|
| 15 |
)
|
|
|
|
| 336 |
|
| 337 |
with gr.TabItem("About 📖", id=3):
|
| 338 |
gr.HTML(ABOUT_TEXT, elem_classes="descriptive-text")
|
| 339 |
+
gr.Markdown(EXAMPLE_CONFIG_TEXT, elem_classes="descriptive-text")
|
| 340 |
|
| 341 |
demo.load(
|
| 342 |
change_tab,
|
src/assets/text_content.py
CHANGED
|
@@ -15,10 +15,55 @@ ABOUT_TEXT = """<h3>About the 🤗 Open LLM-Perf Leaderboard 🏋️</h3>
|
|
| 15 |
<li>Peak memory is measured in MB during the first forward pass of the LLM (no warmup).</li>
|
| 16 |
<li>Each pair of (Model Type, Weight Class) is represented by the best scored model. This LLM is the one used for all the hardware/backend/optimization experiments.</li>
|
| 17 |
<li>Score is the average evaluation score obtained from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a>.</li>
|
| 18 |
-
<li>Ranking is based on the euclidean distance from the "Perfect LLM" (i.e. 0 latency and 100% accuracy).</li>
|
| 19 |
</ul>
|
| 20 |
"""
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results."
|
| 23 |
CITATION_BUTTON_TEXT = r"""@misc{open-llm-perf-leaderboard,
|
| 24 |
author = {Ilyas Moutawwakil, Régis Pierrard},
|
|
|
|
| 15 |
<li>Peak memory is measured in MB during the first forward pass of the LLM (no warmup).</li>
|
| 16 |
<li>Each pair of (Model Type, Weight Class) is represented by the best scored model. This LLM is the one used for all the hardware/backend/optimization experiments.</li>
|
| 17 |
<li>Score is the average evaluation score obtained from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a>.</li>
|
| 18 |
+
<li>Ranking is based on a composite metric which is the euclidean distance from the "Perfect LLM" (i.e. 0 latency and 100% accuracy).</li>
|
| 19 |
</ul>
|
| 20 |
"""
|
| 21 |
|
| 22 |
+
EXAMPLE_CONFIG_TEXT = """
|
| 23 |
+
Here's an example of the configuration file used to benchmark the models with Optimum-Benchmark:
|
| 24 |
+
```yaml
|
| 25 |
+
defaults:
|
| 26 |
+
- backend: pytorch # default backend
|
| 27 |
+
- benchmark: inference # default benchmark
|
| 28 |
+
- experiment # inheriting from experiment config
|
| 29 |
+
- _self_ # for hydra 1.1 compatibility
|
| 30 |
+
- override hydra/job_logging: colorlog # colorful logging
|
| 31 |
+
- override hydra/hydra_logging: colorlog # colorful logging
|
| 32 |
+
|
| 33 |
+
hydra:
|
| 34 |
+
run:
|
| 35 |
+
dir: llm-experiments/{experiment_name}
|
| 36 |
+
job:
|
| 37 |
+
chdir: true
|
| 38 |
+
|
| 39 |
+
experiment_name: {experiment_name}
|
| 40 |
+
|
| 41 |
+
model: {model}
|
| 42 |
+
hub_kwargs:
|
| 43 |
+
revision: {revision}
|
| 44 |
+
trust_remote_code: {trust_remote_code}
|
| 45 |
+
|
| 46 |
+
device: cuda
|
| 47 |
+
|
| 48 |
+
backend:
|
| 49 |
+
no_weights: true
|
| 50 |
+
delete_cache: true
|
| 51 |
+
torch_dtype: float16
|
| 52 |
+
load_in_4bit: true
|
| 53 |
+
bettertransformer: true
|
| 54 |
+
|
| 55 |
+
benchmark:
|
| 56 |
+
memory: true
|
| 57 |
+
|
| 58 |
+
input_shapes:
|
| 59 |
+
batch_size: 1
|
| 60 |
+
sequence_length: 1
|
| 61 |
+
|
| 62 |
+
new_tokens: 1000
|
| 63 |
+
```
|
| 64 |
+
"""
|
| 65 |
+
|
| 66 |
+
|
| 67 |
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results."
|
| 68 |
CITATION_BUTTON_TEXT = r"""@misc{open-llm-perf-leaderboard,
|
| 69 |
author = {Ilyas Moutawwakil, Régis Pierrard},
|