Fix tokenizers (from @unsloth's clean versions)

added_tokens.json

@@ -8,5 +8,6 @@
   "<|tool_call|>": 200025,
   "<|tool_response|>": 200027,
   "<|tool|>": 200023,
-  "<|user|>": 200021
+  "<|user|>": 200021,
+  "<｜PAD▁TOKEN｜>": 200029
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "Phi-4-mini-instruct",
+  "_name_or_path": "microsoft/Phi-4-mini-instruct",
   "architectures": [
     "Phi3ForCausalLM"
   ],
@@ -27,7 +27,7 @@
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
   "original_max_position_embeddings": 4096,
-  "pad_token_id": 199999,
+  "pad_token_id": 200029,
   "partial_rotary_factor": 0.75,
   "resid_pdrop": 0.0,
   "rms_norm_eps": 1e-05,
@@ -138,7 +138,8 @@
   "sliding_window": 262144,
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.45.0",
+  "transformers_version": "4.49.0",
+  "unsloth_fixed": true,
   "use_cache": true,
   "vocab_size": 200064
 }

generation_config.json

@@ -1,7 +1,11 @@
 {
   "_from_model_config": true,
   "bos_token_id": 199999,
-  "eos_token_id": 200020,
+  "eos_token_id": [
+    200020,
+    199999
+  ],
+  "max_length": 131072,
   "pad_token_id": 200029,
-  "transformers_version": "4.45.0"
+  "transformers_version": "4.49.0"
 }

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17271d41abf3fc764f40486fe752ecb6dea75976b86228b4aa75a114434ac993
-size 2768428504
+oid sha256:4b95024d663f331ee51097cf857cafc7159a5b478fc8d35195cb3312b95f101c
+size 2415521792

special_tokens_map.json

@@ -7,17 +7,12 @@
     "single_word": false
   },
   "eos_token": {
-    "content": "<|end|>",
+    "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<|PAD_TOKEN|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
+  "pad_token": "<｜PAD▁TOKEN｜>",
+  "unk_token": "�"
 }

tokenizer_config.json

@@ -3,6 +3,14 @@
   "add_eos_token": false,
   "add_prefix_space": false,
   "added_tokens_decoder": {
+    "3251": {
+      "content": "�",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
     "199999": {
       "content": "<|endoftext|>",
       "lstrip": false,
@@ -100,7 +108,7 @@
       "special": true
     },
     "200029": {
-      "content": "<|PAD▁TOKEN|>",
+      "content": "<｜PAD▁TOKEN｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -111,10 +119,10 @@
   "bos_token": "<|endoftext|>",
   "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|end|>",
+  "eos_token": "<|endoftext|>",
   "extra_special_tokens": {},
   "model_max_length": 131072,
-  "pad_token": "<|PAD_TOKEN|>",
+  "pad_token": "<｜PAD▁TOKEN｜>",
   "padding_side": "left",
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "�"