Upload tokenizer

added_tokens.json

@@ -1,13 +1,17 @@
 {
+  "\"arguments\":": 32014,
+  "\"function\":": 32012,
+  "\"name\":": 32013,
   "<|assistant|>": 32001,
   "<|endoftext|>": 32000,
   "<|end|>": 32007,
+  "<|function_metadata|>": 32008,
   "<|placeholder1|>": 32002,
   "<|placeholder2|>": 32003,
   "<|placeholder3|>": 32004,
-  "<|tool_calls|>": 32005,
-  "<|function_metadata|>": 32008,
-  "<|tool_results|>": 32009,
+  "<|placeholder4|>": 32005,
   "<|system|>": 32006,
+  "<|tool_calls|>": 32011,
+  "<|tool_results|>": 32009,
   "<|user|>": 32010
 }

tokenizer.json

@@ -128,6 +128,42 @@
       "rstrip": true,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 32011,
+      "content": "<|tool_calls|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": true,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32012,
+      "content": "\"function\":",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 32013,
+      "content": "\"name\":",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 32014,
+      "content": "\"arguments\":",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
     }
   ],
   "normalizer": {
@@ -93460,4 +93496,4 @@
       "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
     ]
   }
-}
+}

tokenizer_config.json

@@ -67,7 +67,7 @@
       "special": true
     },
     "32005": {
-      "content": "<|tool_calls|>",
+      "content": "<|placeholder4|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": true,
@@ -113,6 +113,38 @@
       "rstrip": true,
       "single_word": false,
       "special": true
+    },
+    "32011": {
+      "content": "<|tool_calls|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32012": {
+      "content": "\"function\":",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32013": {
+      "content": "\"name\":",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32014": {
+      "content": "\"arguments\":",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
   "bos_token": "<s>",
@@ -128,13 +160,12 @@
   ],
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
-  "legacy": false,
+  "func_bos_token": "<|tool_calls|>",
   "model_max_length": 4096,
   "pad_token": "<|endoftext|>",
   "padding_side": "left",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false,
-  "func_bos_token": "<|tool_calls|>"
+  "use_default_system_prompt": false
 }