from transformers import AutoTokenizer | |
tokenizer = AutoTokenizer.from_pretrained("base-model-name") | |
tokenizer.save_pretrained("./my_model") | |
from transformers import AutoTokenizer | |
tokenizer = AutoTokenizer.from_pretrained("base-model-name") | |
tokenizer.save_pretrained("./my_model") | |