readme

README.md

@@ -37,7 +37,7 @@ Install requirements:
 conda create -n llava_hf python=3.10
 conda activate llava_hf
 conda install pytorch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 pytorch-cuda=11.7 -c pytorch -c nvidia
-pip install requirements.txt
+pip install -r requirements.txt
 ```
 
 Run RaDialog inference:
@@ -66,7 +66,7 @@ def load_model_from_huggingface(repo_id):
     model_path = Path(model_path)
 
     tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, model_base='liuhaotian/llava-v1.5-7b',
-                                                                           model_name="llava-v1.5-7b-task-lora_radialog_instruct_llava_biovil_unfrozen_2e-5_5epochs_v5_checkpoint-21000", load_8bit=False, load_4bit=False)
+                                                                           model_name="llava-v1.5-7b-task-lora_radialog_instruct_llava_biovil_unfrozen_2e-5_5epochs_v5_checkpoint-21000", load_8bit=False, $
 
 
     return tokenizer, model, image_processor, context_len
@@ -74,7 +74,7 @@ def load_model_from_huggingface(repo_id):
 
 
 if __name__ == '__main__':
-    sample_img_path = "https://openi.nlm.nih.gov/imgs/512/10/10/CXR10_IM-0002-2001.png?keywords=Calcified%20Granuloma" #TODO find good image
+    sample_img_path = "https://openi.nlm.nih.gov/imgs/512/294/3502/CXR3502_IM-1707-1001.png?keywords=Surgical%20Instruments,Cardiomegaly,Pulmonary%20Congestion,Diaphragm"
 
     response = requests.get(sample_img_path)
     image = Image.open(io.BytesIO(response.content))
@@ -95,7 +95,7 @@ if __name__ == '__main__':
     findings = ', '.join(findings).lower().strip()
 
     conv = conv_vicuna_v1.copy()
-    REPORT_GEN_PROMPT = f"<image>. Predicted Findings: {findings}. You are to act as a radiologist and write the finding section of a chest x-ray radiology report for this X-ray image and the given predicted findings. Write in the style of a radiologist, write one fluent text without enumeration, be concise and don't provide explanations or reasons."
+    REPORT_GEN_PROMPT = f"<image>. Predicted Findings: {findings}. You are to act as a radiologist and write the finding section of a chest x-ray radiology report for this X-ray image and the given predi$
     print("USER: ", REPORT_GEN_PROMPT)
     conv.append_message("USER", REPORT_GEN_PROMPT)
     conv.append_message("ASSISTANT", None)
@@ -126,6 +126,27 @@ if __name__ == '__main__':
     pred = tokenizer.decode(output_ids[0, input_ids.shape[1]:]).strip().replace("</s>", "")
     print("ASSISTANT: ", pred)
 
+    # add prediction to conversation
+    conv.messages.pop()
+    conv.append_message("ASSISTANT", pred)
+    stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+    stopping_criteria = KeywordsStoppingCriteria([stop_str], tokenizer, input_ids)
+
+    # generate a report
+    with torch.inference_mode():
+        output_ids = model.generate(
+            input_ids,
+            images=image_tensor,
+            do_sample=False,
+            use_cache=True,
+            max_new_tokens=300,
+            stopping_criteria=[stopping_criteria],
+            pad_token_id=tokenizer.pad_token_id
+        )
+
+    pred = tokenizer.decode(output_ids[0, input_ids.shape[1]:]).strip().replace("</s>", "")
+    print("ASSISTANT: ", pred)
+
     # add prediction to conversation
     conv.messages.pop()
     conv.append_message("ASSISTANT", pred)