appready

app.py

@@ -1,6 +1,6 @@
 import subprocess
-# Installing flash_attn
-subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
+# Remove flash-attn installation
+# subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
 
 import gradio as gr
 from PIL import Image 
@@ -13,9 +13,14 @@ import torch
 import spaces
 
 model_id = "microsoft/Phi-3-vision-128k-instruct" 
-model = AutoModelForCausalLM.from_pretrained(model_id, device_map="cpu", trust_remote_code=True, torch_dtype="auto")
+model = AutoModelForCausalLM.from_pretrained(
+    model_id, 
+    device_map="cpu", 
+    trust_remote_code=True, 
+    torch_dtype=torch.float32,  # Explicitly set to float32
+    attn_implementation="eager"  # Disable FlashAttention
+)
 processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True) 
-model.to("cpu")
 
 PLACEHOLDER = """
 <div style="padding: 30px; text-align: center; display: flex; flex-direction: column; align-items: center;">
@@ -30,30 +35,24 @@ def bot_streaming(message, history):
     print(f'message is - {message}')
     print(f'history is - {history}')
     if message["files"]:
-        # message["files"][-1] is a Dict or just a string
         if type(message["files"][-1]) == dict:
             image = message["files"][-1]["path"]
         else:
             image = message["files"][-1]
     else:
-        # if there's no image uploaded for this turn, look for images in the past turns
-        # kept inside tuples, take the last one
         for hist in history:
             if type(hist[0]) == tuple:
                 image = hist[0][0]
     try:
         if image is None:
-            # Handle the case where image is None
             raise gr.Error("You need to upload an image for Phi3-Vision to work. Close the error and try again with an Image.")
     except NameError:
-        # Handle the case where 'image' is not defined at all
         raise gr.Error("You need to upload an image for Phi3-Vision to work. Close the error and try again with an Image.")
 
     conversation = []
     flag=False
     for user, assistant in history:
         if assistant is None:
-            #pass
             flag=True
             conversation.extend([{"role": "user", "content":""}])
             continue
@@ -71,10 +70,17 @@ def bot_streaming(message, history):
     print(f"prompt is -\n{conversation}")
     prompt = processor.tokenizer.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True)
     image = Image.open(image)
-    inputs = processor(prompt, image, return_tensors="pt").to("cpu") 
+    inputs = processor(prompt, image, return_tensors="pt")
 
     streamer = TextIteratorStreamer(processor, **{"skip_special_tokens": True, "skip_prompt": True, 'clean_up_tokenization_spaces':False,}) 
-    generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=1024, do_sample=False, temperature=0.0, eos_token_id=processor.tokenizer.eos_token_id,)
+    generation_kwargs = dict(
+        inputs, 
+        streamer=streamer, 
+        max_new_tokens=1024, 
+        do_sample=False, 
+        temperature=0.0, 
+        eos_token_id=processor.tokenizer.eos_token_id
+    )
 
     thread = Thread(target=model.generate, kwargs=generation_kwargs)
     thread.start()
@@ -84,7 +90,6 @@ def bot_streaming(message, history):
         buffer += new_text
         yield buffer
 
-
 chatbot=gr.Chatbot(scale=1, placeholder=PLACEHOLDER)
 chat_input = gr.MultimodalTextbox(interactive=True, file_types=["image"], placeholder="Enter message or upload file...", show_label=False)
 with gr.Blocks(fill_height=True, ) as demo: