Hugging Face's logo

Chakita
/
None-stage2

Text Generation
Transformers
PyTorch
gpt2
text-generation-inference
Inference Endpoints
Model card Files Community
1
Chakita commited on
Commit
62fa3be
·
1 Parent(s): 7062f03

Tokenizer: Stage 2 - Epoch 1

Browse files
Files changed (6) hide show
  1. added_tokens.json +25 -0
  2. merges.txt +0 -0
  3. special_tokens_map.json +34 -0
  4. tokenizer.json +0 -0
  5. tokenizer_config.json +49 -0
  6. vocab.json +0 -0
added_tokens.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<BRG>": 50259,
3
+ "<|keywordtext|>": 50257,
4
+ "<|padtext|>": 50278,
5
+ "<|questiontext|>": 50258,
6
+ "<|septext|>": 50279,
7
+ "<|startoftext|>": 50277,
8
+ "N_00": 50267,
9
+ "N_01": 50268,
10
+ "N_02": 50269,
11
+ "N_03": 50270,
12
+ "N_04": 50271,
13
+ "N_05": 50272,
14
+ "N_06": 50273,
15
+ "N_07": 50274,
16
+ "N_08": 50275,
17
+ "N_09": 50276,
18
+ "number0": 50260,
19
+ "number1": 50261,
20
+ "number2": 50262,
21
+ "number3": 50263,
22
+ "number4": 50264,
23
+ "number5": 50265,
24
+ "number6": 50266
25
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|keywordtext|>",
4
+ "<|questiontext|>",
5
+ "<BRG>",
6
+ "N_00",
7
+ "N_01",
8
+ "N_02",
9
+ "N_03",
10
+ "N_04",
11
+ "N_05",
12
+ "N_06",
13
+ "N_07",
14
+ "N_08",
15
+ "N_09"
16
+ ],
17
+ "bos_token": "<|startoftext|>",
18
+ "eos_token": {
19
+ "content": "<|endoftext|>",
20
+ "lstrip": false,
21
+ "normalized": true,
22
+ "rstrip": false,
23
+ "single_word": false
24
+ },
25
+ "pad_token": "<|padtext|>",
26
+ "sep_token": "<|septext|>",
27
+ "unk_token": {
28
+ "content": "<|endoftext|>",
29
+ "lstrip": false,
30
+ "normalized": true,
31
+ "rstrip": false,
32
+ "single_word": false
33
+ }
34
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "additional_special_tokens": [
5
+ "<|keywordtext|>",
6
+ "<|questiontext|>",
7
+ "<BRG>",
8
+ "N_00",
9
+ "N_01",
10
+ "N_02",
11
+ "N_03",
12
+ "N_04",
13
+ "N_05",
14
+ "N_06",
15
+ "N_07",
16
+ "N_08",
17
+ "N_09"
18
+ ],
19
+ "bos_token": {
20
+ "__type": "AddedToken",
21
+ "content": "<|endoftext|>",
22
+ "lstrip": false,
23
+ "normalized": true,
24
+ "rstrip": false,
25
+ "single_word": false
26
+ },
27
+ "eos_token": {
28
+ "__type": "AddedToken",
29
+ "content": "<|endoftext|>",
30
+ "lstrip": false,
31
+ "normalized": true,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ },
35
+ "errors": "replace",
36
+ "model_max_length": 1024,
37
+ "name_or_path": "mwp/keybert-lm-solvability-stage1",
38
+ "pad_token": null,
39
+ "special_tokens_map_file": null,
40
+ "tokenizer_class": "GPT2Tokenizer",
41
+ "unk_token": {
42
+ "__type": "AddedToken",
43
+ "content": "<|endoftext|>",
44
+ "lstrip": false,
45
+ "normalized": true,
46
+ "rstrip": false,
47
+ "single_word": false
48
+ }
49
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff