Update README.md
Browse files
README.md
CHANGED
@@ -1,3 +1,4 @@
|
|
|
|
1 |
def DummyTok(model_max_length=4):
|
2 |
import tempfile
|
3 |
|
@@ -13,4 +14,5 @@ def DummyTok(model_max_length=4):
|
|
13 |
)
|
14 |
|
15 |
real_tokenizer.save("dummy_tokenizer.json")
|
16 |
-
return real_tokenizer
|
|
|
|
1 |
+
```python
|
2 |
def DummyTok(model_max_length=4):
|
3 |
import tempfile
|
4 |
|
|
|
14 |
)
|
15 |
|
16 |
real_tokenizer.save("dummy_tokenizer.json")
|
17 |
+
return real_tokenizer
|
18 |
+
```
|