Adding special tokens

app.py

@@ -19,6 +19,7 @@ finetunned_BERT = BertTextGenerator(finetunned_BERT_model_name, tokenizer='bert-
 finetunned_RoBERTa_model_name = "JuanJoseMV/XLM_RoBERTa_text_gen"
 finetunned_RoBERTa = BertTextGenerator(finetunned_RoBERTa_model_name, tokenizer=finetunned_RoBERTa_model_name)
 
+## Add special tokens
 special_tokens = [
     '[POSITIVE-0]', 
     '[POSITIVE-1]', 
@@ -28,6 +29,12 @@ special_tokens = [
     '[NEGATIVE-2]'
     ]
 
+BERT.tokenizer.add_special_tokens({'additional_special_tokens': special_tokens})
+BERT.model.resize_token_embeddings(len(BERT.tokenizer))
+
+RoBERTa.tokenizer.add_special_tokens({'additional_special_tokens': special_tokens})
+RoBERTa.model.resize_token_embeddings(len(RoBERTa.tokenizer))
+
 finetunned_BERT.tokenizer.add_special_tokens({'additional_special_tokens': special_tokens})
 finetunned_BERT.model.resize_token_embeddings(len(finetunned_BERT.tokenizer))