Dec ZeroGPU usage

app.py

@@ -11,7 +11,7 @@ model_path = "Pectics/Softie-VL-7B-250123"
 model = Qwen2VLForConditionalGeneration.from_pretrained(
     model_path,
     torch_dtype=bfloat16,
-    #attn_implementation="flash_attention_2",
+    attn_implementation="flash_attention_2",
     device_map="auto",
 )
 min_pixels = 256 * 28 * 28
@@ -19,19 +19,12 @@ max_pixels = 1280 * 28 * 28
 processor: Qwen2VLProcessor = AutoProcessor.from_pretrained(model_path, min_pixels=min_pixels, max_pixels=max_pixels)
 
 @spaces.GPU
-def respond(
-    message,
-    history,
-    system_message,
+def infer(
+    messages,
     max_tokens,
     temperature,
     top_p,
 ):
-    messages = [{"role": "system", "content": system_message}]
-    for m in history:
-        messages.append({"role": m["role"], "content": m["content"]})
-    messages.append({"role": "user", "content": message})
-    
     text_inputs = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
     image_inputs, video_inputs = process_vision_info(messages)
     inputs = processor(
@@ -58,6 +51,20 @@ def respond(
         response += token
         yield response
 
+def respond(
+    message,
+    history,
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+):
+    messages = [{"role": "system", "content": system_message}]
+    for m in history:
+        messages.append({"role": m["role"], "content": m["content"]})
+    messages.append({"role": "user", "content": message})
+    return infer(messages, max_tokens, temperature, top_p)
+
 app = gr.ChatInterface(
     respond,
     type="messages",

requirements.txt

@@ -3,4 +3,5 @@ torchvision
 transformers
 accelerate
 qwen-vl-utils
-gradio
+gradio
+flash-attn