CKPT: Space CPU version

Dockerfile

@@ -26,4 +26,4 @@ COPY --chown=user . $HOME/app
 
 RUN uv sync --frozen --extra cu124
 
-CMD ["uv","run","app.py","--strategy","cpu i4","--model_title","RWKV-x070-World-0.1B-v2.8-20241210-ctx4096","--download_repo_id","BlinkDL/rwkv-7-world","--host","0.0.0.0","--port","7860"]
+CMD ["uv","run","app.py","--strategy","cuda fp16","--model_title","RWKV-x070-World-0.1B-v2.8-20241210-ctx4096","--download_repo_id","BlinkDL/rwkv-7-world","--host","0.0.0.0","--port","7860"]

app.py

@@ -92,7 +92,7 @@ class ChatCompletionRequest(BaseModel):
         description="Add `:thinking` suffix to the model name to enable reasoning. Example: `rwkv-latest:thinking`",
     )
     messages: List[ChatMessage]
-    prompt: Union[str, None] = Field(default=None)
+    prompt: Optional[str] = Field(default=None)
     max_tokens: int = Field(default=512)
     temperature: float = Field(default=1.0)
     top_p: float = Field(default=0.3)
@@ -114,7 +114,7 @@ app.add_middleware(
 )
 
 
-def runPrefill(ctx: str, model_tokens: List[int], model_state):
+async def runPrefill(ctx: str, model_tokens: List[int], model_state):
     ctx = ctx.replace("\r\n", "\n")
 
     tokens = pipeline.encode(ctx)
@@ -124,6 +124,7 @@ def runPrefill(ctx: str, model_tokens: List[int], model_state):
     while len(tokens) > 0:
         out, model_state = model.forward(tokens[: CONFIG.CHUNK_LEN], model_state)
         tokens = tokens[CONFIG.CHUNK_LEN :]
+        await asyncio.sleep(0)
 
     return out, model_tokens, model_state
 
@@ -220,7 +221,7 @@ async def chatResponse(
         else request.prompt.strip()
     )
 
-    out, model_tokens, model_state = runPrefill(prompt, [], model_state)
+    out, model_tokens, model_state = await runPrefill(prompt, [], model_state)
 
     prefillTime = time.time()
     promptTokenCount = len(model_tokens)
@@ -301,7 +302,7 @@ async def chatResponseStream(
         else request.prompt.strip()
     )
 
-    out, model_tokens, model_state = runPrefill(prompt, [], model_state)
+    out, model_tokens, model_state = await runPrefill(prompt, [], model_state)
 
     prefillTime = time.time()
     promptTokenCount = len(model_tokens)
@@ -530,7 +531,7 @@ async def chat_completions(request: ChatCompletionRequest):
     completionId = str(next(CompletionIdGenerator))
     logger.info(f"[REQ] {completionId} - {request.model_dump()}")
 
-    def chatResponseStreamDisconnect():
+    async def chatResponseStreamDisconnect():
         if "cuda" in CONFIG.STRATEGY:
             gpu_info = nvmlDeviceGetMemoryInfo(gpu_h)
             logger.info(