{ | |
"added_tokens_decoder": { | |
"0": { | |
"content": "<unk>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"1": { | |
"content": "<s>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"2": { | |
"content": "</s>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"92544": { | |
"content": "<table>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92545": { | |
"content": "</table>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92546": { | |
"content": "<tr>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92547": { | |
"content": "</tr>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92548": { | |
"content": "<td>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92549": { | |
"content": "</td>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92550": { | |
"content": "<th>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92551": { | |
"content": "</th>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92552": { | |
"content": "colspan=", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92553": { | |
"content": "rowspan=", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92554": { | |
"content": "<thead>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92555": { | |
"content": "</thead>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92556": { | |
"content": "<tbody>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92557": { | |
"content": "</tbody>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
} | |
}, | |
"auto_map": { | |
"AutoTokenizer": [ | |
"tokenization_internlm_xcomposer2.InternLMXComposer2Tokenizer", | |
null | |
] | |
}, | |
"bos_token": "<s>", | |
"clean_up_tokenization_spaces": false, | |
"eos_token": "</s>", | |
"model_max_length": 1000000000000000019884624838656, | |
"pad_token": "</s>", | |
"padding_side": "right", | |
"tokenizer_class": "InternLMXComposer2Tokenizer", | |
"unk_token": "<unk>", | |
"use_fast": false | |
} | |