Update: integrate llama3 into finetuning code

accelerator_config/gpu_1_config.yml

@@ -0,0 +1,15 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: 'NO'
+downcast_bf16: 'no'
+gpu_ids: '0'
+machine_rank: 0
+main_training_function: main
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 1
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

accelerator_config/gpu_4_config.yml

@@ -0,0 +1,16 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: MULTI_GPU
+downcast_bf16: 'no'
+gpu_ids: 0,1,2,3
+machine_rank: 0
+main_training_function: main
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 4
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false
+main_process_port: 29600

dataset/SurgDataset.py

@@ -28,6 +28,8 @@ class SurgDataset(Dataset):
             if os.path.isfile(args.data_path):
                 with open(args.data_path) as f:
                     self.data_json = json.load(f)
+                if len(self.data_json) > 200:
+                    self.data_json = self.data_json[:200]
             else:
                 self.data_json_path = os.path.join(args.data_path, 'test.json')
                 if os.path.isfile(self.data_json_path):

eval_scripts/caption_eval.py

@@ -0,0 +1,92 @@
+import json
+from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
+from pycocoevalcap.rouge.rouge import Rouge
+from pycocoevalcap.cider.cider import Cider
+import itertools
+def load_json(filename):
+    with open(filename, 'r') as file:
+        return json.load(file)
+
+def extract_answers(llm_data, gt_data):
+    llm_answers = {item['id']: item['answer'] for item in llm_data}
+    gt_answers = {item['id']: [conv['value'] for conv in item['conversations'] if conv['from'] == 'gpt'][0] for item in gt_data}
+    return llm_answers, gt_answers
+
+def compute_bleu_scores(reference, hypothesis):
+    smooth_fn = SmoothingFunction().method1
+    weights = [
+        (1, 0, 0, 0),        # BLEU-1
+        (0.5, 0.5, 0, 0),    # BLEU-2
+        (0.33, 0.33, 0.33, 0), # BLEU-3
+        (0.25, 0.25, 0.25, 0.25)  # BLEU-4
+    ]
+    scores = [sentence_bleu([reference], hypothesis, weights=w, smoothing_function=smooth_fn) for w in weights]
+    return scores
+
+def compute_rouge_scores(references, hypotheses):
+    rouge = Rouge()
+    scores, _ = rouge.compute_score(references, hypotheses)
+    return scores
+
+def compute_cider_scores(references, hypotheses):
+    cider = Cider()
+    scores, _ = cider.compute_score(references, hypotheses)
+    return scores
+
+def main(llm_file, gt_file):
+    llm_data = load_json(llm_file)
+    gt_data = load_json(gt_file)
+    
+    llm_answers, gt_answers = extract_answers(llm_data, gt_data)
+    
+    bleu_scores = {i: [] for i in range(4)}
+    references = {}
+    hypotheses = {}
+    
+    for id in llm_answers:
+        if id in gt_answers:
+            hypothesis = llm_answers[id].split()
+            reference = gt_answers[id].split()
+            bleu = compute_bleu_scores(reference, hypothesis)
+            for i in range(4):
+                bleu_scores[i].append(bleu[i])
+            references[id] = [gt_answers[id]]
+            hypotheses[id] = [llm_answers[id]]
+        else:
+            print(f"ID {id} not found in ground truth data.")
+    
+    rouge_scores = compute_rouge_scores(references, hypotheses)
+    cider_scores = compute_cider_scores(references, hypotheses)
+    
+    avg_bleu_scores = [sum(scores) / len(scores) for scores in bleu_scores.values()]
+    # print(rouge_scores)
+    # print(cider_scores)
+    # avg_rouge_score = sum(rouge_scores) / len(rouge_scores)
+    # avg_cider_score = sum(cider_scores) / len(cider_scores)
+
+    print(f"Average BLEU-1: {avg_bleu_scores[0]:.4f}")
+    print(f"Average BLEU-2: {avg_bleu_scores[1]:.4f}")
+    print(f"Average BLEU-3: {avg_bleu_scores[2]:.4f}")
+    print(f"Average BLEU-4: {avg_bleu_scores[3]:.4f}")
+    print(f"Average ROUGE-L: {rouge_scores:.4f}")
+    print(f"Average CIDEr: {cider_scores:.4f}")
+
+
+def test():
+    llm_type = ['finetune', 'ori']
+    data_gt = {'pwiseg': '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_caption_instruct_0712_test.json',
+                '4dor': '/mnt1/lyc/llava_finetune/data_json/4dor_caption_instruct_0711_test.json'
+                 }
+    data_type = ['pwiseg', '4dor']
+    for dt, lt in itertools.product(data_gt, llm_type):
+     # Replace with your ground truth file path
+        print(f'[INFO] data {dt} llm {lt}')
+        llm_file = f'/mnt1/lyc/llava_finetune/eval_output/results_{dt}_{lt}/preds_description.json'
+        gt_file = data_gt[dt]
+        main(llm_file, gt_file)
+        print()
+        print()
+
+if __name__ == '__main__':
+    test()
+    

eval_scripts/count_eval.py

@@ -0,0 +1,237 @@
+import json
+from openai import OpenAI
+import re
+
+
+def load_json(filename):
+    with open(filename, 'r') as file:
+        return json.load(file)
+
+
+client = OpenAI(api_key="sk-HZLqWTFgQKHUM0YN9d800981DbC34aEa90632493B9310360", 
+                base_url="https://vip.yi-zhan.top/v1")
+
+def get_result(prompt):
+    response = client.chat.completions.create(
+        model="gpt-4o-2024-05-13",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant"},
+            {"role": "user", "content": 
+            [
+                {"type": "text", "text": prompt},
+            ] 
+             },
+        ],
+        stream=False,
+        temperature=0.8
+    )
+    return response.choices[0].message.content
+
+
+def create_prompt(question, llm_answer, gt_answer):
+    
+    template = """
+    ## Role
+    You are a judge, tasked with determining whether the answers provided by other large language models are consistent with the annotated data, especially in terms of numerical accuracy.
+
+    ## Question
+    ```json
+    {question}
+    ```
+
+    ## LLM Answer
+    ```json
+    {llm_answer}
+    ```
+    
+    ## Annotated Answer
+     ```json
+    {gt_answer}
+    ```
+   
+    ## Task
+    For a given Question, evaluate whether the LLM Answer is consistent with the Annotated Answer. If it is, please answer yes and give a reason.If it is not, please answer no and give a reason.
+
+    ## Constraints
+    - Your response should be divided into two parts: 'answer' and 'reason'. The 'answer' should be either 'yes' or 'no', indicating whether the large language model's prediction aligns with the annotated information, particularly in terms of quantities. The 'reason' should provide the rationale for your answer.
+    - When evaluating the accuracy of the large language model's prediction, please pay close attention to the counting of quantities in the model's response and whether it matches the quantities provided in the standard information.
+    - output format is a json dict as follows:
+    "reason": reason,
+    "answer": answer 
+
+    Take a deep breath and start your answer step by step.
+    """
+
+    prompt = template.format(question=question, 
+                             llm_answer=llm_answer, 
+                             gt_answer=gt_answer)
+    return prompt
+
+# def extract_answer(response_text):
+#     pattern = r'"answer":\s*"([^"]+)"'
+#     match = re.search(pattern, response_text)
+#     print(match)
+#     if match:
+#         return match.group(1).lower() == 'yes'
+#     return False
+
+def extract_answer(json_string):
+    # 使用正则表达式匹配answer和reason
+    answer_match = re.search(r'"answer":\s*"([^"]+)"', json_string)
+    reason_match = re.search(r'"reason":\s*"([^"]+)"', json_string, re.DOTALL)
+
+    # 提取匹配的内容
+    answer = answer_match.group(1) if answer_match else None
+    reason = reason_match.group(1) if reason_match else None
+
+    return answer, reason
+
+
+def main(llm_file, gt_file, out_file):
+    llm_data = load_json(llm_file)
+    gt_data = load_json(gt_file)
+    
+    QA_dict = {item["id"]:{} for item in llm_data}
+    
+    for item in llm_data:
+        qid = item["id"]
+        QA_dict[qid]["question"] = item["question"]
+        QA_dict[qid]["llm_answer"] = item["answer"]
+
+    for item in gt_data:
+        qid = item["id"]
+      #   import ipdb
+      #   ipdb.set_trace()
+        if qid in QA_dict.keys():
+            QA_dict[qid]["gt_answer"] = item["conversations"][1]["value"]
+
+    compares = []
+    correct_ans = 0
+    for ix, (qid, item) in enumerate(QA_dict.items()):
+        question   = item["question"]
+        llm_answer = item["llm_answer"]
+        gt_answer  = item["gt_answer"]
+        prompt = create_prompt(question, llm_answer, gt_answer)
+
+        try:
+            compare = get_result(prompt=prompt)
+
+            answer, reason = extract_answer(compare)
+            compare_data = {"id": qid, "answer": answer, "reason":reason}
+            compares.append(compare_data)
+            with open(out_file, 'w') as f:
+                json.dump(compares, f, indent=4)
+            
+            if answer.lower()== 'yes':
+                correct_ans = correct_ans + 1
+                print(f"#correct \n answer:{answer},\n reason:{reason}")
+            else:
+                print(f"#wrong \n answer:{answer},\n reason:{reason}")
+            
+        except:
+            print("break", item)
+            continue
+        print(f"[step {ix}, correct {correct_ans}, total {len(QA_dict)}, rate {correct_ans/len(QA_dict)}")  
+        compares.append(compare)
+    
+    return compares
+
+
+
+if __name__ == "__main__":
+      
+      #################################################
+      #                4dor count                      #
+      #################################################
+
+      # # intern fintuned: [step 200, correct 75, total 200, rate 0.375]
+      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results_eval/4dor_count_instruct_0711_test_compare.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      # # intern origin: step 199, correct 18, total 200, rate 0.09
+      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results_eval/4dor_count_instruct_0711_test_compare.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      # llava 7b fintuned: [step 199, correct 111, total 200, rate 0.555]
+      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results_eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      # ## llava 7b origin: [step 199, correct 44, total 200, rate 0.22]
+      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results-eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      ## llava 13b fintuned: [step 199, correct 125, total 200, rate 0.625]
+      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results_eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      # ## llava 13b origin: [step 199, correct 16, total 200, rate 0.08]
+      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results-eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      #################################################
+      #                pwi count                      #
+      #################################################
+
+      # intern fintuned: step 199, correct 60, total 200, rate 0.3
+      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results_eval/pwiseg_count_instruct_0712_test_compare.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      # # intern origin: step 199, correct 22, total 200, rate 0.11
+      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results_eval/pwiseg_count_instruct_0712_test_results.json'  # 替���为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      # llava 7b fintuned: step 198, correct 140, total 200, rate 0.7
+      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results_eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      # ## llava 7b origin:  step 199, correct 12, total 200, rate 0.06
+      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results-eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      # # # llava 13b fintuned: step 199, correct 142, total 200, rate 0.71
+    #   llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
+    #   gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
+    #   out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
+    #   compares = main(llm_file, gt_file, out_file)
+
+      ## llava 13b origin: [step 199, correct 142, total 200, rate 0.71]
+    #   llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
+    #   gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
+    #   out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results-eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
+    #   compares = main(llm_file, gt_file, out_file)
+    
+      ## LLaVA-NeXT
+      llm_file = '/mnt1/lyc/llava_finetune/eval_output/results_pwiseg_ori/preds_count.json'  # 替换为你的 LLM 预测文件路径
+      gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
+      out_file = '/mnt1/lyc/llava_finetune/eval_output/pwiseg_count_eval_llama3_llava.json'  # 替换为你的 LLM 预测文件路径
+      
+      
+    #   llm_file = '/mnt1/lyc/llava_finetune/eval_output/results_4dor_ori/preds_count.json'  # 替换为你的 LLM 预测文件路径
+    #   gt_file = '/mnt1/lyc/llava_finetune/data_json/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+    #   out_file = '/mnt1/lyc/llava_finetune/eval_output/4dor_count_eval_llama3_llava_ori.json'  # 替换为你的 LLM 预测文件路径
+      
+      compares = main(llm_file, gt_file, out_file)
+ 
+
+
+

eval_scripts/phase_eval.py

@@ -0,0 +1,179 @@
+import json
+from openai import OpenAI
+import re
+
+
+def load_json(filename):
+    with open(filename, 'r') as file:
+        return json.load(file)
+
+
+client = OpenAI(api_key="sk-HZLqWTFgQKHUM0YN9d800981DbC34aEa90632493B9310360", 
+                base_url="https://vip.yi-zhan.top/v1")
+
+def get_result(prompt):
+    response = client.chat.completions.create(
+        model="gpt-4o-2024-05-13",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant"},
+            {"role": "user", "content": 
+            [
+                {"type": "text", "text": prompt},
+            ] 
+             },
+        ],
+        stream=False,
+        temperature=0.8
+    )
+    return response.choices[0].message.content
+
+
+def create_prompt(question, llm_answer, gt_answer):
+    
+    template = """
+        ## Role
+        You are a fair judge, comparing the LLM answer with the annotated answer, and evaluating whether the answer is accurate about the understanding of the surgical stage.
+
+        ## Question
+        ```json
+        {question}
+        ```
+
+        ## LLM answer
+        ```json
+        {llm_answer}
+        ```
+
+        ## Annotated answer
+        ```json
+        {gt_answer}
+        ```
+
+        ## Task
+        For the given question, evaluate whether the LLM answer is consistent with the annotated answer. If yes, answer yes and give a reason. If no, answer no and give a reason.
+
+        ## Constraints
+        - Your answer should be divided into two parts: "Answer" and "Reason". "Answer" should be "Yes" or "No", indicating whether the large language model's prediction is consistent with the annotation information. "Reason" should provide the reason for your answer.
+        - When evaluating the accuracy of the LLM's prediction, pay close attention to whether the model's answer is accurate about the understanding of the surgical phase, including whether the surgical stage is correctly identified and whether the operation suggestion given is appropriate.
+        - Output format is json Dictionary, as shown below:
+        "reason":reason,
+        "answer":answer
+
+        Take a deep breath and start answering step by step.
+        """
+
+
+    prompt = template.format(question=question, 
+                             llm_answer=llm_answer, 
+                             gt_answer=gt_answer)
+    return prompt
+
+
+def extract_answer(json_string):
+    # 使用正则表达式匹配answer和reason
+    answer_match = re.search(r'"answer":\s*"([^"]+)"', json_string)
+    reason_match = re.search(r'"reason":\s*"([^"]+)"', json_string, re.DOTALL)
+
+    # 提取匹配的内容
+    answer = answer_match.group(1) if answer_match else None
+    reason = reason_match.group(1) if reason_match else None
+
+    return answer, reason
+
+
+def main(llm_file, gt_file, out_file):
+    llm_data = load_json(llm_file)
+    gt_data = load_json(gt_file)
+    
+    QA_dict = {item["id"]:{} for item in llm_data}
+    
+    for item in llm_data:
+        qid = item["id"]
+        QA_dict[qid]["question"] = item["question"]
+        QA_dict[qid]["llm_answer"] = item["answer"]
+
+    for item in gt_data:
+        qid = item["id"]
+      #   import ipdb
+      #   ipdb.set_trace()
+        if qid in QA_dict.keys():
+            QA_dict[qid]["gt_answer"] = item["conversations"][1]["value"]
+
+    compares = []
+    correct_ans = 0
+    for ix, (qid, item) in enumerate(QA_dict.items()):
+        question   = item["question"]
+        llm_answer = item["llm_answer"]
+        gt_answer  = item["gt_answer"]
+        prompt = create_prompt(question, llm_answer, gt_answer)
+
+        try:
+            compare = get_result(prompt=prompt)
+
+            answer, reason = extract_answer(compare)
+            compare_data = {"id": qid, "answer": answer, "reason":reason}
+            compares.append(compare_data)
+            with open(out_file, 'w') as f:
+                json.dump(compares, f, indent=4)
+            
+            if answer.lower()== 'yes':
+                correct_ans = correct_ans + 1
+                print(f"#correct \n answer:{answer},\n reason:{reason}")
+            else:
+                print(f"#wrong \n answer:{answer},\n reason:{reason}")
+            
+        except:
+            print("break", item)
+            continue
+        print(f"[step {ix}, correct {correct_ans}, total {len(QA_dict)}, rate {correct_ans/len(QA_dict)}")  
+        compares.append(compare)
+    
+    return compares
+
+
+
+if __name__ == "__main__":
+
+      # # intern fintuned: step 199, correct 42, total 200, rate 0.21
+    #   llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results/4dor_phase_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+    #   gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_phase_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+    #   out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results_eval/4dor_phase_instruct_0711_test_compare.json'  # 替换为你的 LLM 预测文件路径
+    #   compares = main(llm_file, gt_file, out_file)
+
+      # intern origin: 
+    #   llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results/4dor_phase_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+    #   gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_phase_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+    #   out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results_eval/4dor_phase_instruct_0711_test_compare.json'  # 替换为你的 LLM 预测文件路径
+    #   compares = main(llm_file, gt_file, out_file)
+
+      # llava 7b fintuned: [step 199, correct 111, total 200, rate 0.555]
+      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results_eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      # ## llava 7b origin: [step 199, correct 44, total 200, rate 0.22]
+      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results-eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      ## llava 13b fintuned: [step 199, correct 125, total 200, rate 0.625]
+      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results_eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      # compares = main(llm_file, gt_file, out_file)
+
+      # ## llava 13b origin
+      #   llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      #   gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+      #   out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results-eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
+      #   compares = main(llm_file, gt_file, out_file)
+
+
+    llm_file = '/mnt1/lyc/llava_finetune/results_4dor/preds_phase.json'  # 替换为你的 LLM 预测文件路径
+    gt_file = '/mnt1/lyc/llava_finetune/data_json/4dor_phase_instruct_0711_test.json'      # 替换为你的 GT 文件路径
+    out_file = '/mnt1/lyc/llava_finetune/eval_output/4dor_phase_eval.json'  # 替换为你的 LLM 预测文件路径
+    compares = main(llm_file, gt_file, out_file)
+
+

run_finetune_llava.py

@@ -2,7 +2,7 @@ from llava.model.builder import load_pretrained_model
 from llava.mm_utils import get_model_name_from_path, process_images, tokenizer_image_token
 from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX
 from llava.conversation import conv_templates, SeparatorStyle
-from peft import LoraConfig, get_peft_model
+from peft import LoraConfig, get_peft_model, PeftModel
 from PIL import Image
 import requests
 import copy
@@ -10,7 +10,7 @@ import torch
 import argparse 
 from dataset.SurgDataset import SurgDataset
 from accelerate import Accelerator
-from models.SurgLLaVA import SurgLLaVA
+from llava.model.SurgLLaVA import SurgLLaVA
 import os
 from tqdm import tqdm
 import json
@@ -38,9 +38,10 @@ def parse_args():
     parser.add_argument('--step_size', type=int, default=300)
     parser.add_argument('--gamma', type=float, default=0.95, help='gemma value of scheduler')
     parser.add_argument('--num_epochs', type=int, default=1000)
-
+    parser.add_argument('--lora', action='store_true', help='Use LoRA if True')
     parser.add_argument('--test', action='store_true')
-    parser.add_argument('--checkpoint_path', type=str, default=None)
+    parser.add_argument('--lora_ckpt_path', type=str, default=None)
+    parser.add_argument('--ckpt_path', type=str, default=None)
     parser.add_argument('--output_dir', type=str, default='4dor_output', help='output file path, which will store output text.')
     return parser.parse_args()
 def main():
@@ -48,7 +49,9 @@ def main():
     accelerator = Accelerator(project_dir=os.path.join(args.ckpt_dir, args.model_name),
                             log_with="wandb" if args.wandb else None,
                             gradient_accumulation_steps=args.gradient_accumulation_steps)
+    
     if args.wandb:
+        print(f'[INFO] Using wandb for logging...')
         accelerator.init_trackers(
             project_name=args.wandb_project, 
             config=args,
@@ -57,55 +60,67 @@ def main():
         accelerator.print("[Info] Using wandb for logging...")
     pretrained = "lmms-lab/llama3-llava-next-8b"
     model_name = "llava_llama3"
-    tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, device_map='cuda') # Add any other thing you want to pass in llava_model_args
-    tokenizer.pad_token_id = tokenizer.eos_token_id
-    train_dataset = SurgDataset(args, image_processor, model.config, mode='train')
-    test_dataset = SurgDataset(args, image_processor, model.config, mode='test')
+    tokenizer, llm_model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, device_map='cuda') # Add any other thing you want to pass in llava_model_args
+    # tokenizer.pad_token_id = tokenizer.eos_token_id
+    if tokenizer.pad_token is None:
+        tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+    tokenizer.add_special_tokens({'pad_token': tokenizer.eos_token})
+    train_dataset = SurgDataset(args, image_processor, llm_model.config, mode='train')
+    test_dataset = SurgDataset(args, image_processor, llm_model.config, mode='test')
     train_dataloader = torch.utils.data.DataLoader(train_dataset, shuffle=True, batch_size=args.batch_size, num_workers=4)
     test_dataloader = torch.utils.data.DataLoader(test_dataset, shuffle=True, batch_size=args.batch_size, num_workers=4)
-        
-    for param in model.parameters():
+    
+    print(f'[INFO] Freezing llm model')
+    for param in llm_model.parameters():
         param.requires_grad = False
-    model.eval()
-    print(f'[INFO] Using LoRA ...')
-    peft_config = LoraConfig(
-            lora_alpha=args.lora_rank,
-            lora_dropout=0.05,
-            r=args.lora_rank,
-            bias="none",
-            task_type="CAUSAL_LM",
-            target_modules=[
-                "q_proj",
-                "k_proj",
-                "v_proj",
-                "o_proj",
-                "gate_proj",
-                "up_proj",
-                "down_proj",
-                "lm_head",
-                ],
-    )
-    lora_llm = get_peft_model(model, peft_config)
-    model = lora_llm.model
+    llm_model.eval()
 
-    train_params = model.parameters()
+    if args.lora:
+        if args.lora_ckpt_path is not None:
+            print(f'[INFO] Loading LoRA model checkpoint...')
+            llm_model = PeftModel.from_pretrained(llm_model, './model_ckpt/llama3-llava-next-8b-task-lora')
+            llm_model = llm_model.merge_and_unload()
+        else:
+            print(f'[INFO] Creating LoRA ...')
+            peft_config = LoraConfig(
+                    lora_alpha=args.lora_rank,
+                    lora_dropout=0.05,
+                    r=args.lora_rank,
+                    bias="none",
+                    task_type="CAUSAL_LM",
+                    target_modules=[
+                        "q_proj",
+                        "k_proj",
+                        "v_proj",
+                        "o_proj",
+                        "gate_proj",
+                        "up_proj",
+                        "down_proj",
+                        "lm_head",
+                        ],
+            )
+            lora_llm = get_peft_model(llm_model, peft_config)
+            llm_model = lora_llm.model
+        
+
+    train_params = llm_model.parameters()
     print(f'[INFO] Creating Model ...')
-    train_model = SurgLLaVA(args, model, tokenizer)
-    train_model = train_model.to(torch.bfloat16)
+    model = SurgLLaVA(args, llm_model, tokenizer)
+    model = model.to(torch.bfloat16)
     optimizer = torch.optim.AdamW(train_params, lr=args.lr, eps=1e-7)
     scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=len(train_dataloader) * args.step_size // args.gradient_accumulation_steps, gamma=args.gamma)
-    if args.checkpoint_path is not None:
-        # Stupid way to load lora... 
-        # TODO: update this!
-        print(f'[INFO] Load checkpoint...')
-        whole_model = torch.load(os.path.join(args.checkpoint_path, 'pytorch_model.bin'), map_location='cpu')
-        train_model.load_state_dict(whole_model)
 
-    train_model, tokenizer, optimizer, scheduler, train_dataloader, test_dataloader = accelerator.prepare(train_model, tokenizer, optimizer, scheduler, train_dataloader, test_dataloader)
+    if args.ckpt_path is not None:
+        print(f'[INFO] Load whole pretrained checkpoint...')
+        whole_model = torch.load(os.path.join(args.ckpt_path, 'pytorch_model.bin'), map_location='cpu')
+        model.load_state_dict(whole_model)
+
+    print(f'[INFO] Preparing accelerator...')
+    model, tokenizer, optimizer, scheduler, train_dataloader, test_dataloader = accelerator.prepare(model, tokenizer, optimizer, scheduler, train_dataloader, test_dataloader)
     if args.test:
         # testing code
         accelerator.print(f'[INFO] Start testing...')
-        train_model.eval()
+        model.eval()
         with torch.no_grad():
             os.makedirs(args.output_dir, exist_ok=True)
             output_list = []
@@ -116,7 +131,7 @@ def main():
                 image_sizes = image_sizes[0]
                 if len(image_sizes) != args.batch_size:
                     image_sizes = [torch.cat(image_sizes)]
-                output = train_model(image, image_sizes, question)
+                output = model(image, image_sizes, question)
                 text_output = tokenizer.batch_decode(output, skip_special_tokens=True)
                 output_data = raw_data
                 output_data.update({'answer': text_output, 'question': question})
@@ -146,7 +161,7 @@ def main():
         # initialize epoch-level metrics
         accelerator.print(f'[INFO] Start training...')
         for epoch in tqdm(range(args.num_epochs)):
-            train_model.train()
+            model.train()
             total_train_loss = 0
             for i, batch in enumerate(train_dataloader):
                 optimizer.zero_grad()
@@ -155,10 +170,10 @@ def main():
                 image_sizes = image_sizes[0]
                 if len(image_sizes) != args.batch_size:
                     image_sizes = [torch.cat(image_sizes)]
-                output = train_model(image, image_sizes, question, answer)
+                output = model(image, image_sizes, question, answer)
                 loss = output.loss
                 # Accelerator requires all params to involve gradient descend. This 'dummy loss' can avoid this issue.
-                for param in train_model.parameters():
+                for param in model.parameters():
                     loss += param.sum() * 0.0
                 accelerator.backward(loss)
                 optimizer.step()
@@ -177,7 +192,7 @@ def main():
             total_test_loss = None
             if epoch % args.eval_interval == 0:
                 total_test_loss = 0
-                train_model.eval()
+                model.eval()
                 with torch.no_grad():
                     for i, batch in enumerate(test_dataloader):
                         raw_data, question, answer, image, image_sizes = batch
@@ -185,7 +200,7 @@ def main():
                         image_sizes = image_sizes[0]
                         if len(image_sizes) != args.batch_size:
                             image_sizes = [torch.cat(image_sizes)]
-                        output = train_model(image, image_sizes, question, )
+                        output = model(image, image_sizes, question, )
                         text_output = tokenizer.batch_decode(output, skip_special_tokens=True)
                         if i % 100 == 0:
                             img_id = raw_data[0]['id']
@@ -198,7 +213,7 @@ def main():
                 save_model_dir = os.path.join(args.ckpt_dir, args.model_name, 'checkpoints', f'checkpoint_{epoch:05d}')
                 lora_save_dir = os.path.join(args.ckpt_dir, args.model_name, 'lora')
                 accelerator.save_state(save_model_dir, safe_serialization=False, total_limit=5)
-                unwrapped_model = accelerator.unwrap_model(train_model)
+                unwrapped_model = accelerator.unwrap_model(model)
                 unwrapped_model.model.save_pretrained(
                     lora_save_dir,
                     save_function=accelerator.save,

scripts/convert_gqa_for_eval.py

@@ -0,0 +1,18 @@
+import os
+import json
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--src", type=str)
+parser.add_argument("--dst", type=str)
+args = parser.parse_args()
+
+all_answers = []
+for line_idx, line in enumerate(open(args.src)):
+    res = json.loads(line)
+    question_id = res['question_id']
+    text = res['text'].rstrip('.').lower()
+    all_answers.append({"questionId": question_id, "prediction": text})
+
+with open(args.dst, 'w') as f:
+    json.dump(all_answers, f)

scripts/convert_mmbench_for_submission.py

@@ -0,0 +1,27 @@
+import os
+import json
+import argparse
+import pandas as pd
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--annotation-file", type=str, required=True)
+    parser.add_argument("--result-dir", type=str, required=True)
+    parser.add_argument("--upload-dir", type=str, required=True)
+    parser.add_argument("--experiment", type=str, required=True)
+
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    args = get_args()
+
+    df = pd.read_table(args.annotation_file)
+
+    cur_df = df.copy()
+    cur_df = cur_df.drop(columns=['hint', 'category', 'source', 'image', 'comment', 'l2-category'])
+    cur_df.insert(6, 'prediction', None)
+    for pred in open(os.path.join(args.result_dir, f"{args.experiment}.jsonl")):
+        pred = json.loads(pred)
+        cur_df.loc[df['index'] == pred['question_id'], 'prediction'] = pred['text']
+
+    cur_df.to_excel(os.path.join(args.upload_dir, f"{args.experiment}.xlsx"), index=False, engine='openpyxl')

scripts/convert_mmvet_for_eval.py

@@ -0,0 +1,18 @@
+import os
+import json
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--src", type=str)
+parser.add_argument("--dst", type=str)
+args = parser.parse_args()
+
+cur_result = {}
+
+for line in open(args.src):
+    data = json.loads(line)
+    qid = data['question_id']
+    cur_result[f'v1_{qid}'] = data['text']
+
+with open(args.dst, 'w') as f:
+    json.dump(cur_result, f, indent=2)

scripts/convert_seed_for_submission.py

@@ -0,0 +1,74 @@
+import os
+import json
+import argparse
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--annotation-file", type=str)
+    parser.add_argument("--result-file", type=str)
+    parser.add_argument("--result-upload-file", type=str)
+    return parser.parse_args()
+
+
+def eval_single(result_file, eval_only_type=None):
+    results = {}
+    for line in open(result_file):
+        row = json.loads(line)
+        results[row['question_id']] = row
+
+    type_counts = {}
+    correct_counts = {}
+    for question_data in data['questions']:
+        if eval_only_type is not None and question_data['data_type'] != eval_only_type: continue
+        data_type = question_data['question_type_id']
+        type_counts[data_type] = type_counts.get(data_type, 0) + 1
+        try:
+            question_id = int(question_data['question_id'])
+        except:
+            question_id = question_data['question_id']
+        if question_id not in results:
+            correct_counts[data_type] = correct_counts.get(data_type, 0)
+            continue
+        row = results[question_id]
+        if row['text'] == question_data['answer']:
+            correct_counts[data_type] = correct_counts.get(data_type, 0) + 1
+
+    total_count = 0
+    total_correct = 0
+    for data_type in sorted(type_counts.keys()):
+        accuracy = correct_counts[data_type] / type_counts[data_type] * 100
+        if eval_only_type is None:
+            print(f"{ques_type_id_to_name[data_type]}: {accuracy:.2f}%")
+
+        total_count += type_counts[data_type]
+        total_correct += correct_counts[data_type]
+
+    total_accuracy = total_correct / total_count * 100
+    if eval_only_type is None:
+        print(f"Total accuracy: {total_accuracy:.2f}%")
+    else:
+        print(f"{eval_only_type} accuracy: {total_accuracy:.2f}%")
+
+    return results
+
+if __name__ == "__main__":
+    args = get_args()
+    data = json.load(open(args.annotation_file))
+    ques_type_id_to_name = {id:n for n,id in data['question_type'].items()}
+
+    results = eval_single(args.result_file)
+    eval_single(args.result_file, eval_only_type='image')
+    eval_single(args.result_file, eval_only_type='video')
+
+    with open(args.result_upload_file, 'w') as fp:
+        for question in data['questions']:
+            qid = question['question_id']
+            if qid in results:
+                result = results[qid]
+            else:
+                result = results[int(qid)]
+            fp.write(json.dumps({
+                'question_id': qid,
+                'prediction': result['text']
+            }) + '\n')

scripts/convert_sqa_to_llava.py

@@ -0,0 +1,88 @@
+import json
+import os
+import fire
+import re
+from convert_sqa_to_llava_base_prompt import build_prompt_chatbot
+
+
+def convert_to_llava(base_dir, split, prompt_format="QCM-LEA"):
+    split_indices = json.load(open(os.path.join(base_dir, "pid_splits.json")))[split]
+    problems = json.load(open(os.path.join(base_dir, "problems.json")))
+
+    split_problems = build_prompt_chatbot(
+        problems, split_indices, prompt_format,
+        use_caption=False, is_test=False)
+
+    target_format = []
+    for prob_id, (input, output) in split_problems.items():
+        if input.startswith('Question: '):
+            input = input.replace('Question: ', '')
+        if output.startswith('Answer: '):
+            output = output.replace('Answer: ', '')
+
+        raw_prob_data = problems[prob_id]
+        if raw_prob_data['image'] is None:
+            target_format.append({
+                "id": prob_id,
+                "conversations": [
+                    {'from': 'human', 'value': f"{input}"},
+                    {'from': 'gpt', 'value': f"{output}"},
+                ],
+            })
+
+        else:
+            target_format.append({
+                "id": prob_id,
+                "image": os.path.join(prob_id, raw_prob_data['image']),
+                "conversations": [
+                    {'from': 'human', 'value': f"{input}\n<image>"},
+                    {'from': 'gpt', 'value': f"{output}"},
+                ],
+            })
+
+    print(f'Number of samples: {len(target_format)}')
+
+    with open(os.path.join(base_dir, f"llava_{split}_{prompt_format}.json"), "w") as f:
+        json.dump(target_format, f, indent=2)
+
+
+def convert_to_jsonl(base_dir, split, prompt_format="QCM-LEPA"):
+    split_indices = json.load(open(os.path.join(base_dir, "pid_splits.json")))[split]
+    problems = json.load(open(os.path.join(base_dir, "problems.json")))
+
+    split_problems = build_prompt_chatbot(
+        problems, split_indices, prompt_format,
+        use_caption=False, is_test=False)
+
+    writer = open(os.path.join(base_dir, f"scienceqa_{split}_{prompt_format}.jsonl"), "w")
+    for prob_id, (input, output) in split_problems.items():
+        if input.startswith('Question: '):
+            input = input.replace('Question: ', '')
+        if output.startswith('Answer: '):
+            output = output.replace('Answer: ', '')
+
+        raw_prob_data = problems[prob_id]
+        if raw_prob_data['image'] is None:
+            data = {
+                "id": prob_id,
+                "instruction": f"{input}",
+                "output": f"{output}",
+            }
+
+        else:
+            data = {
+                "id": prob_id,
+                "image": os.path.join(prob_id, raw_prob_data['image']),
+                "instruction": f"{input}\n<image>",
+                "output": f"{output}",
+            }
+        writer.write(json.dumps(data) + '\n')
+    writer.close()
+
+
+def main(task, **kwargs):
+    globals()[task](**kwargs)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

scripts/convert_sqa_to_llava_base_prompt.py

@@ -0,0 +1,334 @@
+def get_question_text(problem):
+    question = problem['question']
+    return question
+
+
+def get_context_text(problem, use_caption):
+    txt_context = problem['hint']
+    img_context = problem['caption'] if use_caption else ""
+    context = " ".join([txt_context, img_context]).strip()
+    if context == "":
+        context = "N/A"
+    return context
+
+
+def get_choice_text(probelm, options):
+    choices = probelm['choices']
+    choice_list = []
+    for i, c in enumerate(choices):
+        choice_list.append("({}) {}".format(options[i], c))
+    choice_txt = " ".join(choice_list)
+    #print(choice_txt)
+    return choice_txt
+
+
+def get_answer(problem, options):
+    return options[problem['answer']]
+
+
+def get_lecture_text(problem):
+    # \\n: GPT-3 can generate the lecture with more tokens.
+    lecture = problem['lecture'].replace("\n", "\\n")
+    return lecture
+
+
+def get_solution_text(problem):
+    # \\n: GPT-3 can generate the solution with more tokens
+    solution = problem['solution'].replace("\n", "\\n")
+    return solution
+
+
+def create_one_example_chatbot(format, question, context, choice, answer, lecture, solution, test_example=True):
+
+    input_format, output_format = format.split("-")
+
+    ## Inputs
+    if input_format == "CQM":
+        input = f"Context: {context}\nQuestion: {question}\nOptions: {choice}\n"
+    elif input_format == "QCM":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\n"
+    # upper bound experiment
+    elif input_format == "QCML":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture}\n"
+    elif input_format == "QCME":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {solution}\n"
+    elif input_format == "QCMLE":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture} {solution}\n"
+
+    elif input_format == "QCLM":
+        input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture}\nOptions: {choice}\n"
+    elif input_format == "QCEM":
+        input = f"Question: {question}\nContext: {context}\nBECAUSE: {solution}\nOptions: {choice}\n"
+    elif input_format == "QCLEM":
+        input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture} {solution}\nOptions: {choice}\n"
+
+    # Outputs
+    if test_example:
+        output = "Answer:"
+    elif output_format == 'A':
+        output = f"Answer: The answer is {answer}."
+
+    elif output_format == 'AL':
+        output = f"Answer: The answer is {answer}. BECAUSE: {solution}"
+    elif output_format == 'AE':
+        output = f"Answer: The answer is {answer}. BECAUSE: {lecture}"
+    elif output_format == 'ALE':
+        output = f"Answer: The answer is {answer}. BECAUSE: {lecture} {solution}"
+    elif output_format == 'AEL':
+        output = f"Answer: The answer is {answer}. BECAUSE: {solution} {lecture}"
+
+    elif output_format == 'LA':
+        output = f"Answer: {lecture} The answer is {answer}."
+    elif output_format == 'EA':
+        output = f"Answer: {solution} The answer is {answer}."
+    elif output_format == 'LEA':
+        output = f"Answer: {lecture} {solution} The answer is {answer}."
+    elif output_format == 'ELA':
+        output = f"Answer: {solution} {lecture} The answer is {answer}."
+    elif output_format == 'LEPA':
+        output = ''
+        if len(lecture.strip()) > 0:
+            output += f"LECTURE: {lecture}\n"
+        if len(solution.strip()) > 0:
+            output += f"SOLUTION: {solution}\n"
+        output += '###\n'
+        output += f"ANSWER: {answer}."
+
+    input = input.replace("  ", " ").strip()
+    output = output.replace("  ", " ").strip()
+    if input.endswith("BECAUSE:"):
+        input = input.replace("BECAUSE:", "").strip()
+    if output.endswith("BECAUSE:"):
+        output = output.replace("BECAUSE:", "").strip()
+    return input, output
+
+
+def create_one_example(format, question, context, choice, answer, lecture, solution, test_example=True):
+
+    input_format, output_format = format.split("-")
+
+    ## Inputs
+    if input_format == "CQM":
+        input = f"Context: {context}\nQuestion: {question}\nOptions: {choice}\n"
+    elif input_format == "QCM":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\n"
+    # upper bound experiment
+    elif input_format == "QCML":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture}\n"
+    elif input_format == "QCME":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {solution}\n"
+    elif input_format == "QCMLE":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture} {solution}\n"
+
+    elif input_format == "QCLM":
+        input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture}\nOptions: {choice}\n"
+    elif input_format == "QCEM":
+        input = f"Question: {question}\nContext: {context}\nBECAUSE: {solution}\nOptions: {choice}\n"
+    elif input_format == "QCLEM":
+        input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture} {solution}\nOptions: {choice}\n"
+
+    # Outputs
+    if test_example:
+        output = "Answer:"
+    elif output_format == 'A':
+        output = f"Answer: The answer is {answer}."
+
+    elif output_format == 'AL':
+        output = f"Answer: The answer is {answer}. BECAUSE: {solution}"
+    elif output_format == 'AE':
+        output = f"Answer: The answer is {answer}. BECAUSE: {lecture}"
+    elif output_format == 'ALE':
+        output = f"Answer: The answer is {answer}. BECAUSE: {lecture} {solution}"
+    elif output_format == 'AEL':
+        output = f"Answer: The answer is {answer}. BECAUSE: {solution} {lecture}"
+
+    elif output_format == 'LA':
+        output = f"Answer: {lecture} The answer is {answer}."
+    elif output_format == 'EA':
+        output = f"Answer: {solution} The answer is {answer}."
+    elif output_format == 'LEA':
+        output = f"Answer: {lecture} {solution} The answer is {answer}."
+    elif output_format == 'ELA':
+        output = f"Answer: {solution} {lecture} The answer is {answer}."
+
+    text = input + output
+    text = text.replace("  ", " ").strip()
+    if text.endswith("BECAUSE:"):
+        text = text.replace("BECAUSE:", "").strip()
+    return text
+
+
+
+def create_one_example_gpt4(format, question, context, choice, answer, lecture, solution, test_example=True):
+
+    input_format, output_format = format.split("-")
+
+    ## Inputs
+    if input_format == "CQM":
+        input = f"Context: {context}\nQuestion: {question}\nOptions: {choice}\n"
+    elif input_format == "QCM":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\n"
+    # upper bound experiment
+    elif input_format == "QCML":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture}\n"
+    elif input_format == "QCME":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {solution}\n"
+    elif input_format == "QCMLE":
+        input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture} {solution}\n"
+
+    elif input_format == "QCLM":
+        input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture}\nOptions: {choice}\n"
+    elif input_format == "QCEM":
+        input = f"Question: {question}\nContext: {context}\nBECAUSE: {solution}\nOptions: {choice}\n"
+    elif input_format == "QCLEM":
+        input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture} {solution}\nOptions: {choice}\n"
+
+    # Outputs
+    if test_example:
+        output = "Answer:"
+    elif output_format == 'A':
+        output = f"Answer: The answer is {answer}."
+
+    elif output_format == 'AL':
+        output = f"Answer: The answer is {answer}. BECAUSE: {solution}"
+    elif output_format == 'AE':
+        output = f"Answer: The answer is {answer}. BECAUSE: {lecture}"
+    elif output_format == 'ALE':
+        output = f"Answer: The answer is {answer}. BECAUSE: {lecture} {solution}"
+    elif output_format == 'AEL':
+        output = f"Answer: The answer is {answer}. BECAUSE: {solution} {lecture}"
+
+    elif output_format == 'LA':
+        output = f"Answer: {lecture} The answer is {answer}."
+    elif output_format == 'EA':
+        output = f"Answer: {solution} The answer is {answer}."
+    elif output_format == 'LEA':
+        output = f"Answer: {lecture} {solution} The answer is {answer}."
+    elif output_format == 'ELA':
+        output = f"Answer: {solution} {lecture} The answer is {answer}."
+
+    input = input.replace("  ", " ").strip()
+    output = output.replace("  ", " ").strip()
+    if output.endswith("BECAUSE:"):
+        output = output.replace("BECAUSE:", "").strip()
+
+    user_prompt = {"role": "user", "content": f"Can you explain {input}?"}
+    assistant_prompt = {"role": "assistant", "content": f"{output}"}
+
+    return user_prompt, assistant_prompt
+
+
+def build_prompt_chatbot(problems, shot_qids, prompt_format, use_caption=False, options=["A", "B", "C", "D", "E"], is_test=False):
+    examples = {}
+
+    for qid in shot_qids:
+        question = get_question_text(problems[qid])
+        context = get_context_text(problems[qid], use_caption)
+        choice = get_choice_text(problems[qid], options)
+        answer = get_answer(problems[qid], options)
+        lecture = get_lecture_text(problems[qid]).replace('\\n', '\n')
+        solution = get_solution_text(problems[qid]).replace('\\n', '\n')
+
+        train_example = create_one_example_chatbot(prompt_format,
+                                           question,
+                                           context,
+                                           choice,
+                                           answer,
+                                           lecture,
+                                           solution,
+                                           test_example=is_test)
+        examples[qid] = train_example
+    return examples
+
+
+def build_prompt(problems, shot_qids, test_qid, args):
+
+    examples = []
+
+    # n-shot training examples
+    for qid in shot_qids:
+        question = get_question_text(problems[qid])
+        context = get_context_text(problems[qid], args.use_caption)
+        choice = get_choice_text(problems[qid], args.options)
+        answer = get_answer(problems[qid], args.options)
+        lecture = get_lecture_text(problems[qid])
+        solution = get_solution_text(problems[qid])
+
+        train_example = create_one_example(args.prompt_format,
+                                           question,
+                                           context,
+                                           choice,
+                                           answer,
+                                           lecture,
+                                           solution,
+                                           test_example=False)
+        examples.append(train_example)
+
+    # test example
+    question = get_question_text(problems[test_qid])
+    context = get_context_text(problems[test_qid], args.use_caption)
+    choice = get_choice_text(problems[test_qid], args.options)
+    answer = get_answer(problems[test_qid], args.options)
+    lecture = get_lecture_text(problems[test_qid])
+    solution = get_solution_text(problems[test_qid])
+
+    test_example = create_one_example(args.prompt_format,
+                                      question,
+                                      context,
+                                      choice,
+                                      answer,
+                                      lecture,
+                                      solution,
+                                      test_example=True)
+    examples.append(test_example)
+
+    # create the prompt input
+    prompt_input = '\n\n'.join(examples)
+
+    return prompt_input
+
+
+def build_prompt_gpt4(problems, shot_qids, test_qid, args):
+
+    prompt_array = [{"role": "system", "content": "You are a helpful assistant."}]
+
+    # n-shot training examples
+    for qid in shot_qids:
+        question = get_question_text(problems[qid])
+        context = get_context_text(problems[qid], args.use_caption)
+        choice = get_choice_text(problems[qid], args.options)
+        answer = get_answer(problems[qid], args.options)
+        lecture = get_lecture_text(problems[qid])
+        solution = get_solution_text(problems[qid])
+
+        user_prompt, assistant_prompt = create_one_example_gpt4(args.prompt_format,
+                                           question,
+                                           context,
+                                           choice,
+                                           answer,
+                                           lecture,
+                                           solution,
+                                           test_example=False)
+        prompt_array.append(user_prompt)
+        prompt_array.append(assistant_prompt)
+
+    # test example
+    question = get_question_text(problems[test_qid])
+    context = get_context_text(problems[test_qid], args.use_caption)
+    choice = get_choice_text(problems[test_qid], args.options)
+    answer = get_answer(problems[test_qid], args.options)
+    lecture = get_lecture_text(problems[test_qid])
+    solution = get_solution_text(problems[test_qid])
+
+    user_prompt, assistant_prompt = create_one_example_gpt4(args.prompt_format,
+                                      question,
+                                      context,
+                                      choice,
+                                      answer,
+                                      lecture,
+                                      solution,
+                                      test_example=True)
+    prompt_array.append(user_prompt)
+    prompt_array.append(assistant_prompt)
+
+    return prompt_array

scripts/convert_vizwiz_for_submission.py

@@ -0,0 +1,47 @@
+import os
+import argparse
+import json
+
+from llava.eval.m4c_evaluator import EvalAIAnswerProcessor
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--annotation-file', type=str, required=True)
+    parser.add_argument('--result-file', type=str, required=True)
+    parser.add_argument('--result-upload-file', type=str, required=True)
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+
+    args = parse_args()
+
+    os.makedirs(os.path.dirname(args.result_upload_file), exist_ok=True)
+
+    results = []
+    error_line = 0
+    for line_idx, line in enumerate(open(args.result_file)):
+        try:
+            results.append(json.loads(line))
+        except:
+            error_line += 1
+    results = {x['question_id']: x['text'] for x in results}
+    test_split = [json.loads(line) for line in open(args.annotation_file)]
+    split_ids = set([x['question_id'] for x in test_split])
+
+    print(f'total results: {len(results)}, total split: {len(test_split)}, error_line: {error_line}')
+
+    all_answers = []
+
+    answer_processor = EvalAIAnswerProcessor()
+
+    for x in test_split:
+        assert x['question_id'] in results
+        all_answers.append({
+            'image': x['image'],
+            'answer': answer_processor(results[x['question_id']])
+        })
+
+    with open(args.result_upload_file, 'w') as f:
+        json.dump(all_answers, f)

scripts/convert_vqav2_for_submission.py

@@ -0,0 +1,56 @@
+import os
+import argparse
+import json
+
+from llava.eval.m4c_evaluator import EvalAIAnswerProcessor
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dir', type=str, default="./playground/data/eval/vqav2")
+    parser.add_argument('--ckpt', type=str, required=True)
+    parser.add_argument('--split', type=str, required=True)
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+
+    args = parse_args()
+
+    src = os.path.join(args.dir, 'answers', args.split, args.ckpt, 'merge.jsonl')
+    test_split = os.path.join(args.dir, 'llava_vqav2_mscoco_test2015.jsonl')
+    dst = os.path.join(args.dir, 'answers_upload', args.split, f'{args.ckpt}.json')
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
+
+    results = []
+    error_line = 0
+    for line_idx, line in enumerate(open(src)):
+        try:
+            results.append(json.loads(line))
+        except:
+            error_line += 1
+
+    results = {x['question_id']: x['text'] for x in results}
+    test_split = [json.loads(line) for line in open(test_split)]
+    split_ids = set([x['question_id'] for x in test_split])
+
+    print(f'total results: {len(results)}, total split: {len(test_split)}, error_line: {error_line}')
+
+    all_answers = []
+
+    answer_processor = EvalAIAnswerProcessor()
+
+    for x in test_split:
+        if x['question_id'] not in results:
+            all_answers.append({
+                'question_id': x['question_id'],
+                'answer': ''
+            })
+        else:
+            all_answers.append({
+                'question_id': x['question_id'],
+                'answer': answer_processor(results[x['question_id']])
+            })
+
+    with open(dst, 'w') as f:
+        json.dump(all_answers, open(dst, 'w'))

scripts/extract_mm_projector.py

@@ -0,0 +1,47 @@
+"""
+This is just a utility that I use to extract the projector for quantized models.
+It is NOT necessary at all to train, or run inference/serve demos.
+Use this script ONLY if you fully understand its implications.
+"""
+
+
+import os
+import argparse
+import torch
+import json
+from collections import defaultdict
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Extract MMProjector weights')
+    parser.add_argument('--model-path', type=str, help='model folder')
+    parser.add_argument('--output', type=str, help='output file')
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    keys_to_match = ['mm_projector']
+    ckpt_to_key = defaultdict(list)
+    try:
+        model_indices = json.load(open(os.path.join(args.model_path, 'pytorch_model.bin.index.json')))
+        for k, v in model_indices['weight_map'].items():
+            if any(key_match in k for key_match in keys_to_match):
+                ckpt_to_key[v].append(k)
+    except FileNotFoundError:
+        # Smaller models or model checkpoints saved by DeepSpeed.
+        v = 'pytorch_model.bin'
+        for k in torch.load(os.path.join(args.model_path, v), map_location='cpu').keys():
+            if any(key_match in k for key_match in keys_to_match):
+                ckpt_to_key[v].append(k)
+
+    loaded_weights = {}
+
+    for ckpt_name, weight_keys in ckpt_to_key.items():
+        ckpt = torch.load(os.path.join(args.model_path, ckpt_name), map_location='cpu')
+        for k in weight_keys:
+            loaded_weights[k] = ckpt[k]
+
+    torch.save(loaded_weights, args.output)

scripts/finetune.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
+
+# Uncomment and set the following variables correspondingly to run this script:
+
+################## VICUNA ##################
+# PROMPT_VERSION=v1
+# MODEL_VERSION="vicuna-v1-3-7b"
+################## VICUNA ##################
+
+################## LLaMA-2 ##################
+# PROMPT_VERSION="llava_llama_2"
+# MODEL_VERSION="llama-2-7b-chat"
+################## LLaMA-2 ##################
+
+deepspeed llava/train/train_mem.py \
+    --deepspeed ./scripts/zero2.json \
+    --model_name_or_path ./checkpoints/$MODEL_VERSION \
+    --version $PROMPT_VERSION \
+    --data_path ./playground/data/llava_instruct_80k.json \
+    --image_folder /path/to/coco/train2017 \
+    --vision_tower openai/clip-vit-large-patch14 \
+    --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 True \
+    --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 4 \
+    --lazy_preprocess True \
+    --report_to wandb

scripts/finetune/test_llava.sh

@@ -1,4 +1,9 @@
-accelerate launch --config_file ./hf_config/single_gpu_config.yml \
+accelerate launch --config_file ./accelerator_config/gpu_4_config.yml \
 run_finetune_llava.py \
 --test \
---checkpoint_path ./model_ckpt/llava3_mix_instr/checkpoints/checkpoint_00003 \
+--data_path /mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_caption_instruct_0712_test.json \
+--output_dir ./eval_output/results_pwiseg_ori \
+# --lora_ckpt_path /mnt1/lyc/llava_finetune/model_ckpt/llama3-llava-next-8b-task-lora \
+# --ckpt_path ./model_ckpt/llava3_mix_instr/checkpoints/checkpoint_00003 \
+# --data_path /mnt1/lyc/llava_finetune/data_json/4dor_caption_instruct_0711_test.json \
+# --data_path /mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json

scripts/finetune_full_schedule.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
+
+# Uncomment and set the following variables correspondingly to run this script:
+
+################## VICUNA ##################
+# PROMPT_VERSION=v1
+# MODEL_VERSION="vicuna-v1-3-7b"
+################## VICUNA ##################
+
+################## LLaMA-2 ##################
+# PROMPT_VERSION="llava_llama_2"
+# MODEL_VERSION="llama-2-7b-chat"
+################## LLaMA-2 ##################
+
+deepspeed llava/train/train_mem.py \
+    --deepspeed ./scripts/zero2.json \
+    --model_name_or_path ./checkpoints/$MODEL_VERSION \
+    --version $PROMPT_VERSION \
+    --data_path ./playground/data/llava_instruct_158k.json \
+    --image_folder /path/to/coco/train2017 \
+    --vision_tower openai/clip-vit-large-patch14 \
+    --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 True \
+    --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 4 \
+    --lazy_preprocess True \
+    --report_to wandb

scripts/finetune_lora.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
+
+# Uncomment and set the following variables correspondingly to run this script:
+
+################## VICUNA ##################
+# PROMPT_VERSION=v1
+# MODEL_VERSION="vicuna-v1-3-7b"
+################## VICUNA ##################
+
+################## LLaMA-2 ##################
+# PROMPT_VERSION="llava_llama_2"
+# MODEL_VERSION="llama-2-7b-chat"
+################## LLaMA-2 ##################
+
+deepspeed llava/train/train_mem.py \
+    --deepspeed ./scripts/zero2.json \
+    --lora_enable True \
+    --model_name_or_path ./checkpoints/$MODEL_VERSION \
+    --version $PROMPT_VERSION \
+    --data_path ./playground/data/llava_instruct_80k.json \
+    --image_folder /path/to/coco/train2017 \
+    --vision_tower openai/clip-vit-large-patch14 \
+    --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 True \
+    --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --lazy_preprocess True \
+    --dataloader_num_workers 4 \
+    --report_to wandb

scripts/finetune_lora_my.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
+
+# Uncomment and set the following variables correspondingly to run this script:
+
+################## VICUNA ##################
+# PROMPT_VERSION=v1
+# MODEL_VERSION="vicuna-v1-3-7b"
+################## VICUNA ##################
+
+################## LLaMA-2 ##################
+PROMPT_VERSION="llava_llama_2"
+MODEL_VERSION="llama-2-7b-chat"
+################## LLaMA-2 ##################
+
+deepspeed llava/train/train_mem.py \
+    --deepspeed ./scripts/zero2.json \
+    --lora_enable True \
+    --model_name_or_path ./checkpoints/$MODEL_VERSION \
+    --version $PROMPT_VERSION \
+    --data_path /mnt1/wjl/InternLM-XComposer/data/4D-OR-instruct/llava_3d_0503_train.json \
+    --image_folder /mnt1/wjl/InternLM-XComposer/data/4D-OR-MV \
+    --vision_tower openai/clip-vit-large-patch14 \
+    --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 True \
+    --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --lazy_preprocess True \
+    --dataloader_num_workers 4 \
+    --report_to wandb

scripts/finetune_qlora.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
+
+# Uncomment and set the following variables correspondingly to run this script:
+
+################## VICUNA ##################
+# PROMPT_VERSION=v1
+# MODEL_VERSION="vicuna-v1-3-7b"
+################## VICUNA ##################
+
+################## LLaMA-2 ##################
+# PROMPT_VERSION="llava_llama_2"
+# MODEL_VERSION="llama-2-7b-chat"
+################## LLaMA-2 ##################
+
+deepspeed llava/train/train_mem.py \
+    --deepspeed ./scripts/zero2.json \
+    --lora_enable True \
+    --bits 4 \
+    --model_name_or_path ./checkpoints/$MODEL_VERSION \
+    --version $PROMPT_VERSION \
+    --data_path ./playground/data/llava_instruct_80k.json \
+    --image_folder /path/to/coco/train2017 \
+    --vision_tower openai/clip-vit-large-patch14 \
+    --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 True \
+    --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --lazy_preprocess True \
+    --dataloader_num_workers 4 \
+    --report_to wandb

scripts/finetune_sqa.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
+
+deepspeed llava/train/train_mem.py \
+    --deepspeed ./scripts/zero2.json \
+    --model_name_or_path lmsys/vicuna-13b-v1.3 \
+    --version $PROMPT_VERSION \
+    --data_path /Data/ScienceQA/data/scienceqa/llava_train_QCM-LEA.json \
+    --image_folder /Data/ScienceQA/data/scienceqa/images/train \
+    --vision_tower openai/clip-vit-large-patch14 \
+    --pretrain_mm_mlp_adapter ./checkpoints/huggingface/liuhaotian/llava-pretrain-vicuna-13b-v1.3/mm_projector.bin \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 True \
+    --output_dir ./checkpoints/llava-vicuna-13b-v1.3-pretrain_lcs558k_plain-ScienceQA_QCM_LEA-12e \
+    --num_train_epochs 12 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 4 \
+    --lazy_preprocess True \
+    --report_to wandb

scripts/merge_lora_weights.py

@@ -0,0 +1,22 @@
+import argparse
+from llava.model.builder import load_pretrained_model
+from llava.mm_utils import get_model_name_from_path
+
+
+def merge_lora(args):
+    model_name = get_model_name_from_path(args.model_path)
+    tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, device_map='cpu')
+
+    model.save_pretrained(args.save_model_path)
+    tokenizer.save_pretrained(args.save_model_path)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-path", type=str, required=True)
+    parser.add_argument("--model-base", type=str, required=True)
+    parser.add_argument("--save-model-path", type=str, required=True)
+
+    args = parser.parse_args()
+
+    merge_lora(args)

scripts/pretrain.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
+
+# Uncomment and set the following variables correspondingly to run this script:
+
+# MODEL_VERSION=vicuna-v1-3-7b
+# MODEL_VERSION=llama-2-7b-chat
+
+########### DO NOT CHANGE ###########
+########### USE THIS FOR BOTH ###########
+PROMPT_VERSION=plain
+########### DO NOT CHANGE ###########
+
+deepspeed llava/train/train_mem.py \
+    --deepspeed ./scripts/zero2.json \
+    --model_name_or_path ./checkpoints/$MODEL_VERSION \
+    --version $PROMPT_VERSION \
+    --data_path /path/to/pretrain_data.json \
+    --image_folder /path/to/images \
+    --vision_tower openai/clip-vit-large-patch14 \
+    --tune_mm_mlp_adapter True \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 True \
+    --output_dir ./checkpoints/llava-$MODEL_VERSION-pretrain \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 24000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-3 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 4 \
+    --lazy_preprocess True \
+    --report_to wandb

scripts/pretrain_xformers.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Uncomment and set the following variables correspondingly to run this script:
+
+# MODEL_VERSION=vicuna-v1-3-7b
+# MODEL_VERSION=llama-2-7b-chat
+
+########### DO NOT CHANGE ###########
+########### USE THIS FOR BOTH ###########
+PROMPT_VERSION=plain
+########### DO NOT CHANGE ###########
+
+deepspeed llava/train/train_xformers.py \
+    --deepspeed ./scripts/zero2.json \
+    --model_name_or_path ./checkpoints/$MODEL_VERSION \
+    --version $PROMPT_VERSION \
+    --data_path /path/to/pretrain_data.json \
+    --image_folder /path/to/images \
+    --vision_tower openai/clip-vit-large-patch14 \
+    --tune_mm_mlp_adapter True \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 False \
+    --output_dir ./checkpoints/llava-$MODEL_VERSION-pretrain \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 24000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-3 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 4 \
+    --lazy_preprocess True \
+    --report_to wandb

scripts/sqa_eval_batch.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+CHUNKS=8
+for IDX in {0..7}; do
+    CUDA_VISIBLE_DEVICES=$IDX python -m llava.eval.model_vqa_science \
+        --model-path liuhaotian/llava-lcs558k-scienceqa-vicuna-13b-v1.3 \
+        --question-file ~/haotian/datasets/ScienceQA/data/scienceqa/llava_test_QCM-LEA.json \
+        --image-folder ~/haotian/datasets/ScienceQA/data/scienceqa/images/test \
+        --answers-file ./test_llava-13b-chunk$CHUNKS_$IDX.jsonl \
+        --num-chunks $CHUNKS \
+        --chunk-idx $IDX \
+        --conv-mode llava_v1 &
+done