diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..52373fe24473b1aa44333d318f578ae6bf04b49b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..211c29a9905aace22a447141927e33aaff64e0f7 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,38 @@ +{ + "": 200034, + "": 200019, + "": 200023, + "": 200018, + "": 200016, + "": 200017, + "": 200015, + "": 200021, + "": 200030, + "": 200020, + "": 200028, + "": 200032, + "": 200006, + "": 200002, + "": 200004, + "": 200001, + "": 200003, + "": 200022, + "": 200007, + "": 200025, + "": 200010, + "": 200009, + "": 200008, + "": 200013, + "": 200035, + "": 200014, + "": 200011, + "": 200012, + "": 200000, + "": 200005, + "": 200024, + "": 200029, + "": 200027, + "": 200031, + "