don't strip the prompt for check since we don't strip to tokenize anymore (#650)
Browse files
src/axolotl/prompt_tokenizers.py
CHANGED
|
@@ -81,7 +81,7 @@ class PromptTokenizingStrategy(abc.ABC):
|
|
| 81 |
self, prompt: str, add_eos_token: bool = True, strip_bos_token: bool = False
|
| 82 |
) -> BatchEncoding:
|
| 83 |
result: BatchEncoding
|
| 84 |
-
if not prompt
|
| 85 |
LOG.warning("Empty text requested for tokenization.")
|
| 86 |
result = BatchEncoding(data={"input_ids": [], "attention_mask": []})
|
| 87 |
else:
|
|
|
|
| 81 |
self, prompt: str, add_eos_token: bool = True, strip_bos_token: bool = False
|
| 82 |
) -> BatchEncoding:
|
| 83 |
result: BatchEncoding
|
| 84 |
+
if not prompt:
|
| 85 |
LOG.warning("Empty text requested for tokenization.")
|
| 86 |
result = BatchEncoding(data={"input_ids": [], "attention_mask": []})
|
| 87 |
else:
|