TRY max_split_size_mb:128

app.py

@@ -5,7 +5,7 @@ if os.environ.get("MODELSCOPE_ENVIRONMENT") == "studio":
 
     patch_hub()
 
-os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:256"
+os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:128"
 
 
 from config import CONFIG, ModelConfig
@@ -247,7 +247,7 @@ def generate(
         if token == 0 and token in request.stop_tokens:
             yield {
                 "content": "".join(cache_word_list),
-                "tokens": out_tokens[out_last - cache_word_len :],
+                "tokens": out_tokens[out_last :],
                 "finish_reason": "stop:token:0",
                 "state": model_state,
             }
@@ -266,7 +266,7 @@ def generate(
         if token in request.stop_tokens:
             yield {
                 "content": "".join(cache_word_list),
-                "tokens": out_tokens[out_last - cache_word_len :],
+                "tokens": out_tokens[out_last :],
                 "finish_reason": f"stop:token:{token}",
                 "state": model_state,
             }