KuBERT / added_tokens.json
abdulhade's picture
Upload tokenizer
2b9f6dd verified
{
"<|af|>": 50591,
"<|am|>": 50598,
"<|ar|>": 50536,
"<|as|>": 50614,
"<|az|>": 50568,
"<|ba|>": 50619,
"<|be|>": 50594,
"<|bg|>": 50556,
"<|bn|>": 50566,
"<|bo|>": 50611,
"<|br|>": 50573,
"<|bs|>": 50579,
"<|ca|>": 50534,
"<|cs|>": 50547,
"<|cy|>": 50561,
"<|da|>": 50549,
"<|de|>": 50525,
"<|el|>": 50545,
"<|endoftext|>": 50628,
"<|en|>": 50523,
"<|es|>": 50526,
"<|et|>": 50571,
"<|eu|>": 50574,
"<|fa|>": 50564,
"<|fi|>": 50541,
"<|fo|>": 50602,
"<|fr|>": 50529,
"<|gl|>": 50583,
"<|gu|>": 50597,
"<|haw|>": 50616,
"<|ha|>": 50618,
"<|he|>": 50543,
"<|hi|>": 50540,
"<|hr|>": 50555,
"<|ht|>": 50603,
"<|hu|>": 50550,
"<|hy|>": 50576,
"<|id|>": 50539,
"<|is|>": 50575,
"<|it|>": 50538,
"<|ja|>": 50530,
"<|jw|>": 50620,
"<|ka|>": 50593,
"<|kk|>": 50580,
"<|km|>": 50587,
"<|kn|>": 50570,
"<|ko|>": 50528,
"<|la|>": 50558,
"<|lb|>": 50609,
"<|ln|>": 50617,
"<|lo|>": 50600,
"<|lt|>": 50557,
"<|lv|>": 50565,
"<|mg|>": 50613,
"<|mi|>": 50559,
"<|mk|>": 50572,
"<|ml|>": 50560,
"<|mn|>": 50578,
"<|mr|>": 50584,
"<|ms|>": 50546,
"<|mt|>": 50607,
"<|my|>": 50610,
"<|ne|>": 50577,
"<|nl|>": 50535,
"<|nn|>": 50606,
"<|nocaptions|>": 50626,
"<|notimestamps|>": 50627,
"<|no|>": 50552,
"<|oc|>": 50592,
"<|pa|>": 50585,
"<|pl|>": 50533,
"<|ps|>": 50604,
"<|pt|>": 50531,
"<|ro|>": 50548,
"<|ru|>": 50527,
"<|sa|>": 50608,
"<|sd|>": 50596,
"<|si|>": 50586,
"<|sk|>": 50562,
"<|sl|>": 50569,
"<|sn|>": 50588,
"<|so|>": 50590,
"<|sq|>": 50581,
"<|sr|>": 50567,
"<|startoflm|>": 50624,
"<|startofprev|>": 50625,
"<|startoftranscript|>": 50522,
"<|su|>": 50621,
"<|sv|>": 50537,
"<|sw|>": 50582,
"<|ta|>": 50551,
"<|te|>": 50563,
"<|tg|>": 50595,
"<|th|>": 50553,
"<|tk|>": 50605,
"<|tl|>": 50612,
"<|transcribe|>": 50623,
"<|translate|>": 50622,
"<|tr|>": 50532,
"<|tt|>": 50615,
"<|uk|>": 50544,
"<|ur|>": 50554,
"<|uz|>": 50601,
"<|vi|>": 50542,
"<|yi|>": 50599,
"<|yo|>": 50589,
"<|zh|>": 50524,
"<|¡|>": 50629,
"<|¢|>": 50630,
"<|£|>": 50631,
"<|¤|>": 50632,
"<|¥|>": 50633,
"<|¦|>": 50634,
"<|§|>": 50635,
"<|¨|>": 50636,
"<|©|>": 50637,
"<|ª|>": 50638,
"<|«|>": 50639,
"<|¬|>": 50640,
"<|®|>": 50641,
"<|¯|>": 50642,
"<|°|>": 50643,
"<|±|>": 50644,
"<|²|>": 50645,
"<|³|>": 50646,
"<|´|>": 50647,
"<|µ|>": 50648,
"<|¶|>": 50649,
"<|·|>": 50650,
"<|¸|>": 50651,
"<|¹|>": 50652,
"<|º|>": 50653,
"<|»|>": 50654,
"<|¼|>": 50655,
"<|½|>": 50656,
"<|¾|>": 50657,
"<|¿|>": 50658,
"<|À|>": 50659,
"<|Á|>": 50660,
"<|Â|>": 50661,
"<|Ã|>": 50662,
"<|Ä|>": 50663,
"<|Å|>": 50664,
"<|Æ|>": 50665,
"<|Ç|>": 50666,
"<|È|>": 50667,
"<|É|>": 50668,
"<|Ê|>": 50669,
"<|Ë|>": 50670,
"<|Ì|>": 50671,
"<|Í|>": 50672,
"<|Î|>": 50673,
"<|Ï|>": 50674,
"<|Ð|>": 50675,
"<|Ñ|>": 50676,
"<|Ò|>": 50677,
"<|Ó|>": 50678,
"<|Ô|>": 50679,
"<|Õ|>": 50680,
"<|Ö|>": 50681,
"<|×|>": 50682,
"<|Ø|>": 50683,
"<|ا|>": 50793,
"<|اÙĨ|>": 50801,
"<|ت|>": 50800,
"<|د|>": 50803,
"<|ر|>": 50797,
"<|س|>": 50807,
"<|Ù|>": 50684,
"<|Ùħ|>": 50804,
"<|ÙĨ|>": 50796,
"<|ÙĪ|>": 50795,
"<|Ú|>": 50685,
"<|Ú©|>": 50798,
"<|Û|>": 50686,
"<|ÛĨ|>": 50806,
"<|ÛĮ|>": 50792,
"<|Ûİ|>": 50802,
"<|Ûķ|>": 50791,
"<|Ü|>": 50687,
"<|Ý|>": 50688,
"<|Þ|>": 50689,
"<|ß|>": 50690,
"<|à|>": 50691,
"<|á|>": 50692,
"<|â|>": 50693,
"<|ã|>": 50694,
"<|ä|>": 50695,
"<|å|>": 50696,
"<|æ|>": 50697,
"<|ç|>": 50698,
"<|è|>": 50699,
"<|é|>": 50700,
"<|ê|>": 50701,
"<|ë|>": 50702,
"<|ì|>": 50703,
"<|í|>": 50704,
"<|î|>": 50705,
"<|ï|>": 50706,
"<|ð|>": 50707,
"<|ñ|>": 50708,
"<|ò|>": 50709,
"<|ó|>": 50710,
"<|ô|>": 50711,
"<|õ|>": 50712,
"<|ö|>": 50713,
"<|÷|>": 50714,
"<|ø|>": 50715,
"<|ù|>": 50716,
"<|ú|>": 50717,
"<|û|>": 50718,
"<|ü|>": 50719,
"<|ý|>": 50720,
"<|þ|>": 50721,
"<|ÿ|>": 50722,
"<|Ā|>": 50723,
"<|ā|>": 50724,
"<|Ă|>": 50725,
"<|ă|>": 50726,
"<|Ą|>": 50727,
"<|ą|>": 50728,
"<|Ć|>": 50729,
"<|ć|>": 50730,
"<|Ĉ|>": 50731,
"<|ĉ|>": 50732,
"<|Ċ|>": 50733,
"<|ċ|>": 50734,
"<|Č|>": 50735,
"<|č|>": 50736,
"<|Ď|>": 50737,
"<|ď|>": 50738,
"<|Đ|>": 50739,
"<|đ|>": 50740,
"<|Ē|>": 50741,
"<|ē|>": 50742,
"<|Ĕ|>": 50743,
"<|ĕ|>": 50744,
"<|Ė|>": 50745,
"<|ė|>": 50746,
"<|Ę|>": 50747,
"<|ę|>": 50748,
"<|Ě|>": 50749,
"<|ě|>": 50750,
"<|Ĝ|>": 50751,
"<|ĝ|>": 50752,
"<|Ğ|>": 50753,
"<|ğ|>": 50754,
"<|Ġ|>": 50755,
"<|ĠØ|>": 50794,
"<|Ġب|>": 50805,
"<|ĠÙ|>": 50799,
"<|ġ|>": 50756,
"<|Ģ|>": 50757,
"<|ģ|>": 50758,
"<|Ĥ|>": 50759,
"<|ĥ|>": 50760,
"<|Ħ|>": 50761,
"<|ħ|>": 50762,
"<|Ĩ|>": 50763,
"<|ĩ|>": 50764,
"<|Ī|>": 50765,
"<|ī|>": 50766,
"<|Ĭ|>": 50767,
"<|ĭ|>": 50768,
"<|Į|>": 50769,
"<|į|>": 50770,
"<|İ|>": 50771,
"<|ı|>": 50772,
"<|IJ|>": 50773,
"<|ij|>": 50774,
"<|Ĵ|>": 50775,
"<|ĵ|>": 50776,
"<|Ķ|>": 50777,
"<|ķ|>": 50778,
"<|ĸ|>": 50779,
"<|Ĺ|>": 50780,
"<|ĺ|>": 50781,
"<|Ļ|>": 50782,
"<|ļ|>": 50783,
"<|Ľ|>": 50784,
"<|ľ|>": 50785,
"<|Ŀ|>": 50786,
"<|ŀ|>": 50787,
"<|Ł|>": 50788,
"<|ł|>": 50789,
"<|Ń|>": 50790
}