Update README.md

README.md

@@ -28,11 +28,24 @@ This adapter equips Phi-3 with the ability to perform symbolic music reasoning a
 ```
 from transformers import AutoModelForCausalLM 
 from transformers import AutoProcessor
+from PIL import Image 
+from http import HTTPStatus
+import torch
+import requests
+from io import BytesIO
 
+def load_img(img_dir):
+  if img_dir.startswith('http://') or img_dir.startswith('https://'):
+      response = requests.get(img_dir)
+      image = Image.open(BytesIO(response.content)).convert('RGB')
+  else:
+      image = Image.open(img_dir).convert('RGB')
+  return image
 
-self.model = AutoModelForCausalLM.from_pretrained('microsoft/Phi-3-vision-128k-instruct', device_map="cuda", trust_remote_code=True, torch_dtype="auto")
-self.processor = AutoProcessor.from_pretrained('microsoft/Phi-3-vision-128k-instruct', trust_remote_code=True)
-self.model.load_adapter('puar-playground/Phi-3-MusiX')
+
+model = AutoModelForCausalLM.from_pretrained('microsoft/Phi-3-vision-128k-instruct', device_map="cuda", trust_remote_code=True, torch_dtype="auto")
+processor = AutoProcessor.from_pretrained('microsoft/Phi-3-vision-128k-instruct', trust_remote_code=True)
+model.load_adapter('puar-playground/Phi-3-MusiX')
 
 
 prompt = '' + f'USER: Answer the question:\n{question_string}. ASSISTANT:'
@@ -41,10 +54,10 @@ prompt = '' + f'USER: Answer the question:\n{question_string}. ASSISTANT:'
 messages = [{"role": "user", "content": f"<|image_1|>\n{prompt}"}]
 
 # load image from dir
-image = self.load_img(img_dir)
+image = load_img(img_dir)
 
-prompt_in = self.processor.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-inputs = self.processor(prompt_in, [image], return_tensors="pt").to("cuda")
+prompt_in = processor.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+inputs = processor(prompt_in, [image], return_tensors="pt").to("cuda")
 
 generation_args = { 
     "max_new_tokens": 500,
@@ -53,11 +66,11 @@ generation_args = {
 }
 
 with torch.no_grad():
-    generate_ids = self.model.generate(**inputs, eos_token_id=self.processor.tokenizer.eos_token_id, **generation_args)
+    generate_ids = model.generate(**inputs, eos_token_id=processor.tokenizer.eos_token_id, **generation_args)
 
 # remove input tokens
 generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]
-model_answer = self.processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+model_answer = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
         
 ```