Sigurdur
/

ice-tokenizer

Model card Files Files and versions Community

Sigurdur commited on Nov 21, 2023

Commit

724d804

·

1 Parent(s): 8312f51

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -16,7 +16,7 @@ Integrate this tokenizer into your NLP pipeline for preprocessing Icelandic text
 from transformers import GPT2Tokenizer
 # Load the tokenizer
-tokenizer = GPT2Tokenizer.from_pretrained("Sigurdur/ice-gpt")
 tokenizer.pad_token_id = tokenizer.eos_token_id
 tokenizer("Halló heimur!")["input_ids"]

 from transformers import GPT2Tokenizer
 # Load the tokenizer
+tokenizer = GPT2Tokenizer.from_pretrained("Sigurdur/ice-tokenizer")
 tokenizer.pad_token_id = tokenizer.eos_token_id
 tokenizer("Halló heimur!")["input_ids"]