Upload 38 files

.gitattributes

@@ -33,3 +33,32 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l0_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l1_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l10_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l11_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l12_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l13_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l14_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l15_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l16_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l17_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l18_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l19_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l2_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l20_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l21_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l22_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l23_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l24_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l25_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l26_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l27_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l3_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l4_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l5_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l6_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l7_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l8_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l9_together.axmodel filter=lfs diff=lfs merge=lfs -text
+TinySwallow-1.5B-Instruct-ax630c/qwen2_post.axmodel filter=lfs diff=lfs merge=lfs -text

TinySwallow-1.5B-Instruct-ax630c/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

TinySwallow-1.5B-Instruct-ax630c/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.41.2"
+}

TinySwallow-1.5B-Instruct-ax630c/model.embed_tokens.weight.bfloat16.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47be7228eb1738b5051685c8ded39e45ca67d05bb5e857862715da278a9dad55
+size 466747392

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l0_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d95dabd8391d529cd7bf1f3c352d25846af7acbe5643dbd09c27d4b7e31c9ec5
+size 49919320

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l10_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4eb93f516dacd565759b8bdca746563d946aea5c816805729d3dd0b182ca7058
+size 49919512

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l11_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b19a9eb01b0eecb8916cd6ba6b265531ce922d504d329c93ebb37ed9a36d13fa
+size 49919240

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l12_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08b066a9197ca710f020ead56d4bc1b573e335d3e00c273ff3376839a34d0d97
+size 49919480

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l13_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40f3d1d302e44e1a223f4aba7f9f657c952cfb539731596a6fdf72879ee0d099
+size 49919416

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l14_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27a88a9a0489525fba109df6e35614378477aa0925aac140b320494cb1914d56
+size 49919392

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l15_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbbafab16e20e9b5452aafa88522af079915707d76366c3023ae967c3bd40370
+size 49919592

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l16_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e45bec764790d636b2631d5b31c6a5e28c57834a32d52bd8c048d02bb462dbee
+size 49919192

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l17_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a79b21091a45d2fbe32e9bbb07138b0c637b950923c692d067f94a6953d98c2
+size 49919520

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l18_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64b44f4f93b7cb6e449236e7c39695d0071721ae4fdaf0f80384af0f33b38e2b
+size 49919392

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l19_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ead3c4b30d4d8e718fc8fa8fbcdb8a8d24263b12585bfc9ceb9e027840ad000f
+size 49919448

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l1_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c3a930dd1ad67d0a8fb5ff27e5f1f7e099d60caa99bfcc8a4a5896356dac1d1
+size 49919288

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l20_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a65f8db05117af695a7dafb91b433a0ffc870408738d0990710e311330a7ef3
+size 49919088

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l21_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64ff558fac2efaa4b1643b0e745a28b24db3f7a24540da82a0c8847184cf810e
+size 49919576

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l22_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa279b5931436da9fae5fe7e9515dffd8ca28f8a7a9a22035b6b77acbab1c98d
+size 49919096

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l23_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e860c6201b693b38b37ab516ee9a82a4304d4118b1757317e4b89e24a9d2d12f
+size 49919304

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l24_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ceb27bdbca0b2961f79a4dbbba229d6d766eb3b5f187b389c65dc32e3fbf209b
+size 49919560

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l25_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:719726adeafb46dbac082e5886c3c12fa694eb12fac35dd90aa30f103ca5cf91
+size 49919496

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l26_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3107e976552557b691b0bbec2d2c071aed93ba949ec45b144d32d768915de44
+size 49919144

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l27_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73460b8859cecd4140821098aa529e91fd53ba79d74d1eb35a39334941840e62
+size 49919568

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l2_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52911398c6ce896c71e0dcd51f77976f2d9f7733aaddff7f2ae9c82476b1aef0
+size 49919472

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l3_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d78128086596a7f701ba02cb643ceae7367ee5186bfae4e36ad5e690ce95e0a3
+size 49919592

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l4_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28d5e139444b7d0e2cf982a20c8180f0d5893c2b59a9b5117a07ac504def956f
+size 49919456

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l5_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79fd918c39cacce0a6c7dd3ad95bac154d315606c27c05d59c2efebfdc6c6965
+size 49919360

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l6_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a71e3e8eb37a83e383765cda0f8dd697ed7b01084d4a54ed7d181e52040e00f9
+size 49919648

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l7_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0912faada8ff476a5fe3d9245db7b74d4bbf97614715a3e364cd0d920fd3f625
+size 49919344

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l8_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1c622748d1adbff170767a86d117b6ae2f50b9de9eb84ed533d3e6caf5567e2
+size 49919240

TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l9_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2273a2d59a5d695caf171657f154e8da4efccb68426d9bb2c68f990b1bc3a2c9
+size 49919560

TinySwallow-1.5B-Instruct-ax630c/qwen2_post.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c880e8c94c155d9e28a36a729af91cab9d0947ff63033a995889fa4f93cd4a4
+size 237152504

TinySwallow-1.5B-Instruct-ax630c/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

TinySwallow-1.5B-Instruct-ax630c/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'あなたは、Sakana AI株式会社が開発したTinySwallowです。小型ながら、誠実で優秀なアシスタントです。' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nあなたは、Sakana AI株式会社が開発したTinySwallowです。小型ながら、誠実で優秀なアシスタントです。<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

run_TinySwallow_1.5B_prefill_ax630c.sh

@@ -0,0 +1,14 @@
+./main_prefill \
+--template_filename_axmodel "TinySwallow-1.5B-Instruct-ax630c/qwen2_p128_l%d_together.axmodel" \
+--axmodel_num 28 \
+--tokenizer_type 2 \
+--filename_tokenizer_model "http://localhost:8080" \
+--bos 0 --eos 0 \
+--filename_post_axmodel "TinySwallow-1.5B-Instruct-ax630c/qwen2_post.axmodel" \
+--filename_tokens_embed "TinySwallow-1.5B-Instruct-ax630c/model.embed_tokens.weight.bfloat16.bin" \
+--tokens_embed_num 151936 \
+--tokens_embed_size 1536 \
+--use_mmap_load_embed 1 \
+--live_print 1 \
+--continue 1 \
+--prompt "$1"

tinyswallow_tokenizer.py

@@ -0,0 +1,120 @@
+# license: bsd-3-clause
+# origin: https://huggingface.co/AXERA-TECH/DeepSeek-R1-Distill-Qwen-1.5B/blob/main/deepseek-r1_tokenizer.py
+
+from transformers import AutoTokenizer
+from http.server import HTTPServer, BaseHTTPRequestHandler
+import json
+import argparse
+
+
+class Tokenizer_Http:
+    def __init__(self):
+        model_id = "TinySwallow-1.5B-Instruct-ax630c"
+        self.tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+    def encode(self, prompt):
+        messages = [
+            {"role": "system", "content": "あなたは、Sakana AI株式会社が開発したTinySwallowです。小型ながら、誠実で優秀なアシスタントです。"},
+            {"role": "user", "content": prompt}
+        ]
+        text = self.tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+        print(text)
+        token_ids = self.tokenizer.encode(text)
+        return token_ids
+
+    def decode(self, token_ids):
+        return self.tokenizer.decode(token_ids)
+
+    @property
+    def bos_id(self):
+        return self.tokenizer.bos_token_id
+
+    @property
+    def eos_id(self):
+        return self.tokenizer.eos_token_id
+
+    @property
+    def bos_token(self):
+        return self.tokenizer.bos_token
+
+    @property
+    def eos_token(self):
+        return self.tokenizer.eos_token
+
+
+tokenizer = Tokenizer_Http()
+
+print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token)
+print(tokenizer.encode("hello world"))
+
+
+class Request(BaseHTTPRequestHandler):
+    timeout = 5
+    server_version = 'Apache'
+
+    def _set_headers(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json; charset=utf-8")
+        self.end_headers()
+
+    def do_GET(self):
+        print(self.path)
+        self._set_headers()
+
+        if self.path == '/bos_id':
+            bos_id = tokenizer.bos_id
+            msg = json.dumps({'bos_id': bos_id if bos_id is not None else -1}, ensure_ascii=False)
+        elif self.path == '/eos_id':
+            eos_id = tokenizer.eos_id
+            msg = json.dumps({'eos_id': eos_id if eos_id is not None else -1}, ensure_ascii=False)
+        else:
+            msg = json.dumps({'error': 'Invalid path'}, ensure_ascii=False)
+
+        print(msg)
+        self.wfile.write(msg.encode('utf-8'))
+
+    def do_POST(self):
+        content_length = int(self.headers['Content-Length'])
+        data = self.rfile.read(content_length).decode('utf-8')
+
+        self._set_headers()
+
+        try:
+            req = json.loads(data)
+        except json.JSONDecodeError:
+            msg = json.dumps({'error': 'Invalid JSON'}, ensure_ascii=False)
+            self.wfile.write(msg.encode('utf-8'))
+            return
+
+        if self.path == '/encode':
+            prompt = req.get('text', '')
+            token_ids = tokenizer.encode(prompt)
+            msg = json.dumps({'token_ids': token_ids if token_ids else -1}, ensure_ascii=False)
+
+        elif self.path == '/decode':
+            token_ids = req.get('token_ids', [])
+            text = tokenizer.decode(token_ids)
+            msg = json.dumps({'text': text if text else ""}, ensure_ascii=False)
+
+        else:
+            msg = json.dumps({'error': 'Invalid path'}, ensure_ascii=False)
+
+        print(msg)
+        self.wfile.write(msg.encode('utf-8'))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--host', type=str, default='localhost')
+    parser.add_argument('--port', type=int, default=8080)
+    args = parser.parse_args()
+
+    host = (args.host, args.port)
+    print(f'http://{host[0]}:{host[1]}')
+    server = HTTPServer(host, Request)
+    server.serve_forever()
+