flash-attn

app.py

@@ -11,11 +11,11 @@ model_path = "microsoft/Phi-4-multimodal-instruct"
 processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
 model = AutoModelForCausalLM.from_pretrained(
     model_path, 
-    device_map="auto", 
+    device_map="cuda", 
     torch_dtype="auto", 
     trust_remote_code=True, 
-    attn_implementation='eager',
-)
+    attn_implementation='flash_attention_2',
+).cuda()
 
 generation_config = GenerationConfig.from_pretrained(model_path)
 

requirements.txt

@@ -7,5 +7,6 @@ scipy
 soundfile
 pillow
 accelerate
+flash-attn
 transformers
 backoff