eval_scripts/count_eval.py

import json
from openai import OpenAI
import re


def load_json(filename):
    with open(filename, 'r') as file:
        return json.load(file)


client = OpenAI(api_key="sk-HZLqWTFgQKHUM0YN9d800981DbC34aEa90632493B9310360", 
                base_url="https://vip.yi-zhan.top/v1")

def get_result(prompt):
    response = client.chat.completions.create(
        model="gpt-4o-2024-05-13",
        messages=[
            {"role": "system", "content": "You are a helpful assistant"},
            {"role": "user", "content": 
            [
                {"type": "text", "text": prompt},
            ] 
             },
        ],
        stream=False,
        temperature=0.8
    )
    return response.choices[0].message.content


def create_prompt(question, llm_answer, gt_answer):
    
    template = """
    ## Role
    You are a judge, tasked with determining whether the answers provided by other large language models are consistent with the annotated data, especially in terms of numerical accuracy.

    ## Question
    ```json
    {question}
    ```

    ## LLM Answer
    ```json
    {llm_answer}
    ```
    
    ## Annotated Answer
     ```json
    {gt_answer}
    ```
   
    ## Task
    For a given Question, evaluate whether the LLM Answer is consistent with the Annotated Answer. If it is, please answer yes and give a reason.If it is not, please answer no and give a reason.

    ## Constraints
    - Your response should be divided into two parts: 'answer' and 'reason'. The 'answer' should be either 'yes' or 'no', indicating whether the large language model's prediction aligns with the annotated information, particularly in terms of quantities. The 'reason' should provide the rationale for your answer.
    - When evaluating the accuracy of the large language model's prediction, please pay close attention to the counting of quantities in the model's response and whether it matches the quantities provided in the standard information.
    - output format is a json dict as follows:
    "reason": reason,
    "answer": answer 

    Take a deep breath and start your answer step by step.
    """

    prompt = template.format(question=question, 
                             llm_answer=llm_answer, 
                             gt_answer=gt_answer)
    return prompt

# def extract_answer(response_text):
#     pattern = r'"answer":\s*"([^"]+)"'
#     match = re.search(pattern, response_text)
#     print(match)
#     if match:
#         return match.group(1).lower() == 'yes'
#     return False

def extract_answer(json_string):
    # 使用正则表达式匹配answer和reason
    answer_match = re.search(r'"answer":\s*"([^"]+)"', json_string)
    reason_match = re.search(r'"reason":\s*"([^"]+)"', json_string, re.DOTALL)

    # 提取匹配的内容
    answer = answer_match.group(1) if answer_match else None
    reason = reason_match.group(1) if reason_match else None

    return answer, reason


def main(llm_file, gt_file, out_file):
    llm_data = load_json(llm_file)
    gt_data = load_json(gt_file)
    
    QA_dict = {item["id"]:{} for item in llm_data}
    
    for item in llm_data:
        qid = item["id"]
        QA_dict[qid]["question"] = item["question"]
        QA_dict[qid]["llm_answer"] = item["answer"]

    for item in gt_data:
        qid = item["id"]
      #   import ipdb
      #   ipdb.set_trace()
        if qid in QA_dict.keys():
            QA_dict[qid]["gt_answer"] = item["conversations"][1]["value"]

    compares = []
    correct_ans = 0
    for ix, (qid, item) in enumerate(QA_dict.items()):
        question   = item["question"]
        llm_answer = item["llm_answer"]
        gt_answer  = item["gt_answer"]
        prompt = create_prompt(question, llm_answer, gt_answer)

        try:
            compare = get_result(prompt=prompt)

            answer, reason = extract_answer(compare)
            compare_data = {"id": qid, "answer": answer, "reason":reason}
            compares.append(compare_data)
            with open(out_file, 'w') as f:
                json.dump(compares, f, indent=4)
            
            if answer.lower()== 'yes':
                correct_ans = correct_ans + 1
                print(f"#correct \n answer:{answer},\n reason:{reason}")
            else:
                print(f"#wrong \n answer:{answer},\n reason:{reason}")
            
        except:
            print("break", item)
            continue
        print(f"[step {ix}, correct {correct_ans}, total {len(QA_dict)}, rate {correct_ans/len(QA_dict)}")  
        compares.append(compare)
    
    return compares



if __name__ == "__main__":
      
      #################################################
      #                4dor count                      #
      #################################################

      # # intern fintuned: [step 200, correct 75, total 200, rate 0.375]
      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results_eval/4dor_count_instruct_0711_test_compare.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # # intern origin: step 199, correct 18, total 200, rate 0.09
      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results_eval/4dor_count_instruct_0711_test_compare.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # llava 7b fintuned: [step 199, correct 111, total 200, rate 0.555]
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results_eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # ## llava 7b origin: [step 199, correct 44, total 200, rate 0.22]
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results-eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      ## llava 13b fintuned: [step 199, correct 125, total 200, rate 0.625]
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results_eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # ## llava 13b origin: [step 199, correct 16, total 200, rate 0.08]
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results-eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      #################################################
      #                pwi count                      #
      #################################################

      # intern fintuned: step 199, correct 60, total 200, rate 0.3
      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results_eval/pwiseg_count_instruct_0712_test_compare.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # # intern origin: step 199, correct 22, total 200, rate 0.11
      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results_eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # llava 7b fintuned: step 198, correct 140, total 200, rate 0.7
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results_eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # ## llava 7b origin:  step 199, correct 12, total 200, rate 0.06
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results-eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # # # llava 13b fintuned: step 199, correct 142, total 200, rate 0.71
    #   llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
    #   gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
    #   out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
    #   compares = main(llm_file, gt_file, out_file)

      ## llava 13b origin: [step 199, correct 142, total 200, rate 0.71]
    #   llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
    #   gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
    #   out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results-eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
    #   compares = main(llm_file, gt_file, out_file)
    
      ## LLaVA-NeXT
      llm_file = '/mnt1/lyc/llava_finetune/eval_output/results_pwiseg_ori/preds_count.json'  # 替换为你的 LLM 预测文件路径
      gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
      out_file = '/mnt1/lyc/llava_finetune/eval_output/pwiseg_count_eval_llama3_llava.json'  # 替换为你的 LLM 预测文件路径
      
      
    #   llm_file = '/mnt1/lyc/llava_finetune/eval_output/results_4dor_ori/preds_count.json'  # 替换为你的 LLM 预测文件路径
    #   gt_file = '/mnt1/lyc/llava_finetune/data_json/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
    #   out_file = '/mnt1/lyc/llava_finetune/eval_output/4dor_count_eval_llama3_llava_ori.json'  # 替换为你的 LLM 预测文件路径
      
      compares = main(llm_file, gt_file, out_file)