deepvk
/

USER-bge-m3

Sentence Similarity

sentence-transformers

feature-extraction

text-embeddings-inference

Model card Files Files and versions

Tokenizer fix

#1

by BorisTM - opened Jul 8, 2024

base: refs/heads/main

←

from: refs/pr/1

Discussion Files changed

Files changed (1) hide show

tokenizer.json +5 -5

tokenizer.json CHANGED Viewed

@@ -85,8 +85,8 @@
   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
-    "prepend_scheme": "always",
-    "split": true
   },
   "post_processor": {
     "type": "TemplateProcessing",
@@ -172,8 +172,8 @@
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
-    "prepend_scheme": "always",
-    "split": true
   },
   "model": {
     "type": "Unigram",
@@ -184846,4 +184846,4 @@
     ],
     "byte_fallback": false
   }
-}

   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
+    "add_prefix_space": true,
+    "prepend_scheme": "always"
   },
   "post_processor": {
     "type": "TemplateProcessing",
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
+    "add_prefix_space": true,
+    "prepend_scheme": "always"
   },
   "model": {
     "type": "Unigram",
     ],
     "byte_fallback": false
   }
+}