Mitchins commited on
Commit
800ab55
·
1 Parent(s): 05564c5

Fix: Corrected extra_id token indexing

Browse files
Files changed (2) hide show
  1. added_tokens.json +101 -101
  2. vocab.json +0 -0
added_tokens.json CHANGED
@@ -1,102 +1,102 @@
1
  {
2
- "<extra_id_0>": 32099,
3
- "<extra_id_10>": 32089,
4
- "<extra_id_11>": 32088,
5
- "<extra_id_12>": 32087,
6
- "<extra_id_13>": 32086,
7
- "<extra_id_14>": 32085,
8
- "<extra_id_15>": 32084,
9
- "<extra_id_16>": 32083,
10
- "<extra_id_17>": 32082,
11
- "<extra_id_18>": 32081,
12
- "<extra_id_19>": 32080,
13
- "<extra_id_1>": 32098,
14
- "<extra_id_20>": 32079,
15
- "<extra_id_21>": 32078,
16
- "<extra_id_22>": 32077,
17
- "<extra_id_23>": 32076,
18
- "<extra_id_24>": 32075,
19
- "<extra_id_25>": 32074,
20
- "<extra_id_26>": 32073,
21
- "<extra_id_27>": 32072,
22
- "<extra_id_28>": 32071,
23
- "<extra_id_29>": 32070,
24
- "<extra_id_2>": 32097,
25
- "<extra_id_30>": 32069,
26
- "<extra_id_31>": 32068,
27
- "<extra_id_32>": 32067,
28
- "<extra_id_33>": 32066,
29
- "<extra_id_34>": 32065,
30
- "<extra_id_35>": 32064,
31
- "<extra_id_36>": 32063,
32
- "<extra_id_37>": 32062,
33
- "<extra_id_38>": 32061,
34
- "<extra_id_39>": 32060,
35
- "<extra_id_3>": 32096,
36
- "<extra_id_40>": 32059,
37
- "<extra_id_41>": 32058,
38
- "<extra_id_42>": 32057,
39
- "<extra_id_43>": 32056,
40
- "<extra_id_44>": 32055,
41
- "<extra_id_45>": 32054,
42
- "<extra_id_46>": 32053,
43
- "<extra_id_47>": 32052,
44
- "<extra_id_48>": 32051,
45
- "<extra_id_49>": 32050,
46
- "<extra_id_4>": 32095,
47
- "<extra_id_50>": 32049,
48
- "<extra_id_51>": 32048,
49
- "<extra_id_52>": 32047,
50
- "<extra_id_53>": 32046,
51
- "<extra_id_54>": 32045,
52
- "<extra_id_55>": 32044,
53
- "<extra_id_56>": 32043,
54
- "<extra_id_57>": 32042,
55
- "<extra_id_58>": 32041,
56
- "<extra_id_59>": 32040,
57
- "<extra_id_5>": 32094,
58
- "<extra_id_60>": 32039,
59
- "<extra_id_61>": 32038,
60
- "<extra_id_62>": 32037,
61
- "<extra_id_63>": 32036,
62
- "<extra_id_64>": 32035,
63
- "<extra_id_65>": 32034,
64
- "<extra_id_66>": 32033,
65
- "<extra_id_67>": 32032,
66
- "<extra_id_68>": 32031,
67
- "<extra_id_69>": 32030,
68
- "<extra_id_6>": 32093,
69
- "<extra_id_70>": 32029,
70
- "<extra_id_71>": 32028,
71
- "<extra_id_72>": 32027,
72
- "<extra_id_73>": 32026,
73
- "<extra_id_74>": 32025,
74
- "<extra_id_75>": 32024,
75
- "<extra_id_76>": 32023,
76
- "<extra_id_77>": 32022,
77
- "<extra_id_78>": 32021,
78
- "<extra_id_79>": 32020,
79
- "<extra_id_7>": 32092,
80
- "<extra_id_80>": 32019,
81
- "<extra_id_81>": 32018,
82
- "<extra_id_82>": 32017,
83
- "<extra_id_83>": 32016,
84
- "<extra_id_84>": 32015,
85
- "<extra_id_85>": 32014,
86
- "<extra_id_86>": 32013,
87
- "<extra_id_87>": 32012,
88
- "<extra_id_88>": 32011,
89
- "<extra_id_89>": 32010,
90
- "<extra_id_8>": 32091,
91
- "<extra_id_90>": 32009,
92
- "<extra_id_91>": 32008,
93
- "<extra_id_92>": 32007,
94
- "<extra_id_93>": 32006,
95
- "<extra_id_94>": 32005,
96
- "<extra_id_95>": 32004,
97
- "<extra_id_96>": 32003,
98
- "<extra_id_97>": 32002,
99
- "<extra_id_98>": 32001,
100
- "<extra_id_99>": 32000,
101
- "<extra_id_9>": 32090
102
- }
 
1
  {
2
+ "<extra_id_0>": 32000,
3
+ "<extra_id_1>": 32001,
4
+ "<extra_id_2>": 32002,
5
+ "<extra_id_3>": 32003,
6
+ "<extra_id_4>": 32004,
7
+ "<extra_id_5>": 32005,
8
+ "<extra_id_6>": 32006,
9
+ "<extra_id_7>": 32007,
10
+ "<extra_id_8>": 32008,
11
+ "<extra_id_9>": 32009,
12
+ "<extra_id_10>": 32010,
13
+ "<extra_id_11>": 32011,
14
+ "<extra_id_12>": 32012,
15
+ "<extra_id_13>": 32013,
16
+ "<extra_id_14>": 32014,
17
+ "<extra_id_15>": 32015,
18
+ "<extra_id_16>": 32016,
19
+ "<extra_id_17>": 32017,
20
+ "<extra_id_18>": 32018,
21
+ "<extra_id_19>": 32019,
22
+ "<extra_id_20>": 32020,
23
+ "<extra_id_21>": 32021,
24
+ "<extra_id_22>": 32022,
25
+ "<extra_id_23>": 32023,
26
+ "<extra_id_24>": 32024,
27
+ "<extra_id_25>": 32025,
28
+ "<extra_id_26>": 32026,
29
+ "<extra_id_27>": 32027,
30
+ "<extra_id_28>": 32028,
31
+ "<extra_id_29>": 32029,
32
+ "<extra_id_30>": 32030,
33
+ "<extra_id_31>": 32031,
34
+ "<extra_id_32>": 32032,
35
+ "<extra_id_33>": 32033,
36
+ "<extra_id_34>": 32034,
37
+ "<extra_id_35>": 32035,
38
+ "<extra_id_36>": 32036,
39
+ "<extra_id_37>": 32037,
40
+ "<extra_id_38>": 32038,
41
+ "<extra_id_39>": 32039,
42
+ "<extra_id_40>": 32040,
43
+ "<extra_id_41>": 32041,
44
+ "<extra_id_42>": 32042,
45
+ "<extra_id_43>": 32043,
46
+ "<extra_id_44>": 32044,
47
+ "<extra_id_45>": 32045,
48
+ "<extra_id_46>": 32046,
49
+ "<extra_id_47>": 32047,
50
+ "<extra_id_48>": 32048,
51
+ "<extra_id_49>": 32049,
52
+ "<extra_id_50>": 32050,
53
+ "<extra_id_51>": 32051,
54
+ "<extra_id_52>": 32052,
55
+ "<extra_id_53>": 32053,
56
+ "<extra_id_54>": 32054,
57
+ "<extra_id_55>": 32055,
58
+ "<extra_id_56>": 32056,
59
+ "<extra_id_57>": 32057,
60
+ "<extra_id_58>": 32058,
61
+ "<extra_id_59>": 32059,
62
+ "<extra_id_60>": 32060,
63
+ "<extra_id_61>": 32061,
64
+ "<extra_id_62>": 32062,
65
+ "<extra_id_63>": 32063,
66
+ "<extra_id_64>": 32064,
67
+ "<extra_id_65>": 32065,
68
+ "<extra_id_66>": 32066,
69
+ "<extra_id_67>": 32067,
70
+ "<extra_id_68>": 32068,
71
+ "<extra_id_69>": 32069,
72
+ "<extra_id_70>": 32070,
73
+ "<extra_id_71>": 32071,
74
+ "<extra_id_72>": 32072,
75
+ "<extra_id_73>": 32073,
76
+ "<extra_id_74>": 32074,
77
+ "<extra_id_75>": 32075,
78
+ "<extra_id_76>": 32076,
79
+ "<extra_id_77>": 32077,
80
+ "<extra_id_78>": 32078,
81
+ "<extra_id_79>": 32079,
82
+ "<extra_id_80>": 32080,
83
+ "<extra_id_81>": 32081,
84
+ "<extra_id_82>": 32082,
85
+ "<extra_id_83>": 32083,
86
+ "<extra_id_84>": 32084,
87
+ "<extra_id_85>": 32085,
88
+ "<extra_id_86>": 32086,
89
+ "<extra_id_87>": 32087,
90
+ "<extra_id_88>": 32088,
91
+ "<extra_id_89>": 32089,
92
+ "<extra_id_90>": 32090,
93
+ "<extra_id_91>": 32091,
94
+ "<extra_id_92>": 32092,
95
+ "<extra_id_93>": 32093,
96
+ "<extra_id_94>": 32094,
97
+ "<extra_id_95>": 32095,
98
+ "<extra_id_96>": 32096,
99
+ "<extra_id_97>": 32097,
100
+ "<extra_id_98>": 32098,
101
+ "<extra_id_99>": 32099
102
+ }
vocab.json CHANGED
The diff for this file is too large to render. See raw diff