Update README.md
Browse files
README.md
CHANGED
|
@@ -108,7 +108,7 @@ _Note: first prompt may be slower as there is a slight warmup time_
|
|
| 108 |
#### Install
|
| 109 |
|
| 110 |
```bash
|
| 111 |
-
git clone
|
| 112 |
(cd fms-extras && pip install -e .)
|
| 113 |
pip install transformers==4.35.0 sentencepiece numpy
|
| 114 |
```
|
|
@@ -126,7 +126,7 @@ python fms-extras/scripts/paged_speculative_inference.py \
|
|
| 126 |
--tokenizer=$MODEL_PATH \
|
| 127 |
--speculator_path=ibm/granite-7b-lab-accelerator \
|
| 128 |
--speculator_source=hf \
|
| 129 |
-
--speculator_variant=
|
| 130 |
--top_k_tokens_per_head=4,3,2,2,2 \
|
| 131 |
--compile \
|
| 132 |
--compile_mode=reduce-overhead
|
|
@@ -143,7 +143,7 @@ python fms-extras/scripts/paged_speculative_inference.py \
|
|
| 143 |
--tokenizer=$MODEL_PATH \
|
| 144 |
--speculator_path=ibm/granite-7b-lab-accelerator \
|
| 145 |
--speculator_source=hf \
|
| 146 |
-
--speculator_variant=
|
| 147 |
--top_k_tokens_per_head=4,3,2,2,2 \
|
| 148 |
--compile \
|
| 149 |
```
|
|
@@ -159,7 +159,7 @@ python fms-extras/scripts/paged_speculative_inference.py \
|
|
| 159 |
--tokenizer=$MODEL_PATH \
|
| 160 |
--speculator_path=ibm/granite-7b-lab-accelerator \
|
| 161 |
--speculator_source=hf \
|
| 162 |
-
--speculator_variant=
|
| 163 |
--top_k_tokens_per_head=4,3,2,2,2 \
|
| 164 |
--batch_input \
|
| 165 |
--compile \
|
|
|
|
| 108 |
#### Install
|
| 109 |
|
| 110 |
```bash
|
| 111 |
+
git clone https://github.com/foundation-model-stack/fms-extras
|
| 112 |
(cd fms-extras && pip install -e .)
|
| 113 |
pip install transformers==4.35.0 sentencepiece numpy
|
| 114 |
```
|
|
|
|
| 126 |
--tokenizer=$MODEL_PATH \
|
| 127 |
--speculator_path=ibm/granite-7b-lab-accelerator \
|
| 128 |
--speculator_source=hf \
|
| 129 |
+
--speculator_variant=1_4b \
|
| 130 |
--top_k_tokens_per_head=4,3,2,2,2 \
|
| 131 |
--compile \
|
| 132 |
--compile_mode=reduce-overhead
|
|
|
|
| 143 |
--tokenizer=$MODEL_PATH \
|
| 144 |
--speculator_path=ibm/granite-7b-lab-accelerator \
|
| 145 |
--speculator_source=hf \
|
| 146 |
+
--speculator_variant=1_4b \
|
| 147 |
--top_k_tokens_per_head=4,3,2,2,2 \
|
| 148 |
--compile \
|
| 149 |
```
|
|
|
|
| 159 |
--tokenizer=$MODEL_PATH \
|
| 160 |
--speculator_path=ibm/granite-7b-lab-accelerator \
|
| 161 |
--speculator_source=hf \
|
| 162 |
+
--speculator_variant=1_4b \
|
| 163 |
--top_k_tokens_per_head=4,3,2,2,2 \
|
| 164 |
--batch_input \
|
| 165 |
--compile \
|