Hugging Face's logo

weathon
/
3d2smiles_pretrain

Image-Text-to-Text
Transformers
TensorBoard
Safetensors
git
Generated from Trainer
Inference Endpoints
Model card Files Metrics Community
weathon commited on
Commit
f645363
ยท
verified ยท
1 Parent(s): 776ee5f

Upload processor

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +15 -22
  2. tokenizer_config.json +60 -29
special_tokens_map.json CHANGED
@@ -1,43 +1,36 @@
1
  {
2
- "bos_token": {
3
- "content": "๎€€",
4
- "lstrip": false,
5
- "normalized": true,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
  "cls_token": {
10
- "content": "๎€€",
11
  "lstrip": false,
12
- "normalized": true,
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
- "eos_token": {
17
- "content": "๎€",
18
  "lstrip": false,
19
- "normalized": true,
20
  "rstrip": false,
21
  "single_word": false
22
  },
23
- "mask_token": {
24
- "content": "๎€ƒ",
25
- "lstrip": true,
26
- "normalized": true,
27
  "rstrip": false,
28
  "single_word": false
29
  },
30
- "pad_token": {
31
- "content": "\u0000",
32
  "lstrip": false,
33
- "normalized": true,
34
  "rstrip": false,
35
  "single_word": false
36
  },
37
- "sep_token": {
38
- "content": "๎€",
39
  "lstrip": false,
40
- "normalized": true,
41
  "rstrip": false,
42
  "single_word": false
43
  }
 
1
  {
 
 
 
 
 
 
 
2
  "cls_token": {
3
+ "content": "[CLS]",
4
  "lstrip": false,
5
+ "normalized": false,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
+ "mask_token": {
10
+ "content": "[MASK]",
11
  "lstrip": false,
12
+ "normalized": false,
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
+ "pad_token": {
17
+ "content": "[PAD]",
18
+ "lstrip": false,
19
+ "normalized": false,
20
  "rstrip": false,
21
  "single_word": false
22
  },
23
+ "sep_token": {
24
+ "content": "[SEP]",
25
  "lstrip": false,
26
+ "normalized": false,
27
  "rstrip": false,
28
  "single_word": false
29
  },
30
+ "unk_token": {
31
+ "content": "[UNK]",
32
  "lstrip": false,
33
+ "normalized": false,
34
  "rstrip": false,
35
  "single_word": false
36
  }
tokenizer_config.json CHANGED
@@ -1,30 +1,61 @@
1
  {
2
- "char_ords": [
3
- 108,
4
- 35,
5
- 41,
6
- 61,
7
- 51,
8
- 70,
9
- 40,
10
- 93,
11
- 52,
12
- 43,
13
- 79,
14
- 49,
15
- 54,
16
- 78,
17
- 66,
18
- 83,
19
- 46,
20
- 50,
21
- 72,
22
- 45,
23
- 114,
24
- 67,
25
- 80,
26
- 91,
27
- 53
28
- ],
29
- "model_max_length": 1024
30
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "[PAD]",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "100": {
12
+ "content": "[UNK]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "101": {
20
+ "content": "[CLS]",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "102": {
28
+ "content": "[SEP]",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "103": {
36
+ "content": "[MASK]",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "clean_up_tokenization_spaces": false,
45
+ "cls_token": "[CLS]",
46
+ "do_lower_case": true,
47
+ "extra_special_tokens": {},
48
+ "mask_token": "[MASK]",
49
+ "model_input_names": [
50
+ "input_ids",
51
+ "attention_mask"
52
+ ],
53
+ "model_max_length": 512,
54
+ "pad_token": "[PAD]",
55
+ "processor_class": "GitProcessor",
56
+ "sep_token": "[SEP]",
57
+ "strip_accents": null,
58
+ "tokenize_chinese_chars": true,
59
+ "tokenizer_class": "BertTokenizer",
60
+ "unk_token": "[UNK]"
61
+ }