tavbert-ar / tokenizer.json
omrikeren's picture
Added files
96b589c
raw
history blame
4.52 kB
{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Split","pattern":{"String":""},"behavior":"Isolated","invert":false},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[0],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[2],"tokens":["[SEP]"]}}},"decoder":null,"model":{"type":"WordLevel","vocab":{"[CLS]": 0, "[PAD]": 1, "[SEP]": 2, "[UNK]": 3, "unused0": 4, "unused1": 5, "unused2": 6, "unused3": 7, "unused4": 8, "unused5": 9, "unused6": 10, "unused7": 11, "unused8": 12, "unused9": 13, "unused10": 14, "unused11": 15, "unused12": 16, "unused13": 17, "unused14": 18, "unused15": 19, "unused16": 20, "unused17": 21, "unused18": 22, "unused19": 23, "unused20": 24, "unused21": 25, "unused22": 26, "unused23": 27, "unused24": 28, "unused25": 29, "unused26": 30, "unused27": 31, "unused28": 32, "unused29": 33, "unused30": 34, "unused31": 35, "unused32": 36, "unused33": 37, "unused34": 38, "unused35": 39, "unused36": 40, "unused37": 41, "unused38": 42, "unused39": 43, "unused40": 44, "unused41": 45, "unused42": 46, "unused43": 47, "unused44": 48, "unused45": 49, "unused46": 50, "unused47": 51, "unused48": 52, "unused49": 53, "unused50": 54, "unused51": 55, "unused52": 56, "unused53": 57, "unused54": 58, "unused55": 59, "unused56": 60, "unused57": 61, "unused58": 62, "unused59": 63, "unused60": 64, "unused61": 65, "unused62": 66, "unused63": 67, "unused64": 68, "unused65": 69, "unused66": 70, "unused67": 71, "unused68": 72, "unused69": 73, "unused70": 74, "unused71": 75, "unused72": 76, "unused73": 77, "unused74": 78, "unused75": 79, "unused76": 80, "unused77": 81, "unused78": 82, "unused79": 83, "unused80": 84, "unused81": 85, "unused82": 86, "unused83": 87, "unused84": 88, "unused85": 89, "unused86": 90, "unused87": 91, "unused88": 92, "unused89": 93, "unused90": 94, "unused91": 95, "unused92": 96, "unused93": 97, "unused94": 98, "unused95": 99, "unused96": 100, "unused97": 101, "unused98": 102, "unused99": 103, " ": 104, "!": 105, "\"": 106, "#": 107, "$": 108, "%": 109, "&": 110, "'": 111, "(": 112, ")": 113, "*": 114, "+": 115, ",": 116, "-": 117, ".": 118, "/": 119, "0": 120, "1": 121, "2": 122, "3": 123, "4": 124, "5": 125, "6": 126, "7": 127, "8": 128, "9": 129, ":": 130, ";": 131, "<": 132, "=": 133, ">": 134, "?": 135, "@": 136, "A": 137, "B": 138, "C": 139, "D": 140, "E": 141, "F": 142, "G": 143, "H": 144, "I": 145, "J": 146, "K": 147, "L": 148, "M": 149, "N": 150, "O": 151, "P": 152, "Q": 153, "R": 154, "S": 155, "T": 156, "U": 157, "V": 158, "W": 159, "X": 160, "Y": 161, "Z": 162, "[": 163, "\\": 164, "]": 165, "^": 166, "_": 167, "a": 168, "b": 169, "c": 170, "d": 171, "e": 172, "f": 173, "g": 174, "h": 175, "i": 176, "j": 177, "k": 178, "l": 179, "m": 180, "n": 181, "o": 182, "p": 183, "q": 184, "r": 185, "s": 186, "t": 187, "u": 188, "v": 189, "w": 190, "x": 191, "y": 192, "z": 193, "{": 194, "|": 195, "}": 196, "~": 197, "«": 198, "°": 199, "·": 200, "»": 201, "é": 202, "а": 203, "в": 204, "д": 205, "е": 206, "и": 207, "к": 208, "л": 209, "м": 210, "н": 211, "о": 212, "п": 213, "р": 214, "с": 215, "т": 216, "،": 217, "؛": 218, "؟": 219, "ء": 220, "آ": 221, "أ": 222, "ؤ": 223, "إ": 224, "ئ": 225, "ا": 226, "ب": 227, "ة": 228, "ت": 229, "ث": 230, "ج": 231, "ح": 232, "خ": 233, "د": 234, "ذ": 235, "ر": 236, "ز": 237, "س": 238, "ش": 239, "ص": 240, "ض": 241, "ط": 242, "ظ": 243, "ع": 244, "غ": 245, "ـ": 246, "ف": 247, "ق": 248, "ك": 249, "ل": 250, "م": 251, "ن": 252, "ه": 253, "و": 254, "ى": 255, "ي": 256, "ً": 257, "ٌ": 258, "ٍ": 259, "َ": 260, "ُ": 261, "ِ": 262, "ّ": 263, "ْ": 264, "٠": 265, "١": 266, "٢": 267, "٣": 268, "٤": 269, "٥": 270, "٦": 271, "٧": 272, "٨": 273, "٩": 274, "٪": 275, "پ": 276, "ک": 277, "گ": 278, "ھ": 279, "ی": 280, "​": 281, "‌": 282, "‎": 283, "‏": 284, "–": 285, "—": 286, "‘": 287, "’": 288, "“": 289, "”": 290, "•": 291, "…": 292, "‪": 293, "‫": 294, "‬": 295, "‭": 296, "‮": 297, "﴾": 298, "﴿": 299, "�": 300, "[MASK]": 301},"unk_token":"[UNK]"}}