Hugging Face's logo

lyclyc52
/
llava_finetune

Model card Files Community
llava_finetune
File size: 13,462 Bytes 
157f5b2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
 
import json
from openai import OpenAI
import re


def load_json(filename):
    with open(filename, 'r') as file:
        return json.load(file)


client = OpenAI(api_key="sk-HZLqWTFgQKHUM0YN9d800981DbC34aEa90632493B9310360", 
                base_url="https://vip.yi-zhan.top/v1")

def get_result(prompt):
    response = client.chat.completions.create(
        model="gpt-4o-2024-05-13",
        messages=[
            {"role": "system", "content": "You are a helpful assistant"},
            {"role": "user", "content": 
            [
                {"type": "text", "text": prompt},
            ] 
             },
        ],
        stream=False,
        temperature=0.8
    )
    return response.choices[0].message.content


def create_prompt(question, llm_answer, gt_answer):
    
    template = """
    ## Role
    You are a judge, tasked with determining whether the answers provided by other large language models are consistent with the annotated data, especially in terms of numerical accuracy.

    ## Question
    ```json
    {question}
    ```

    ## LLM Answer
    ```json
    {llm_answer}
    ```
    
    ## Annotated Answer
     ```json
    {gt_answer}
    ```
   
    ## Task
    For a given Question, evaluate whether the LLM Answer is consistent with the Annotated Answer. If it is, please answer yes and give a reason.If it is not, please answer no and give a reason.

    ## Constraints
    - Your response should be divided into two parts: 'answer' and 'reason'. The 'answer' should be either 'yes' or 'no', indicating whether the large language model's prediction aligns with the annotated information, particularly in terms of quantities. The 'reason' should provide the rationale for your answer.
    - When evaluating the accuracy of the large language model's prediction, please pay close attention to the counting of quantities in the model's response and whether it matches the quantities provided in the standard information.
    - output format is a json dict as follows:
    "reason": reason,
    "answer": answer 

    Take a deep breath and start your answer step by step.
    """

    prompt = template.format(question=question, 
                             llm_answer=llm_answer, 
                             gt_answer=gt_answer)
    return prompt

# def extract_answer(response_text):
#     pattern = r'"answer":\s*"([^"]+)"'
#     match = re.search(pattern, response_text)
#     print(match)
#     if match:
#         return match.group(1).lower() == 'yes'
#     return False

def extract_answer(json_string):
    # 使用正则表达式匹配answer和reason
    answer_match = re.search(r'"answer":\s*"([^"]+)"', json_string)
    reason_match = re.search(r'"reason":\s*"([^"]+)"', json_string, re.DOTALL)

    # 提取匹配的内容
    answer = answer_match.group(1) if answer_match else None
    reason = reason_match.group(1) if reason_match else None

    return answer, reason


def main(llm_file, gt_file, out_file):
    llm_data = load_json(llm_file)
    gt_data = load_json(gt_file)
    
    QA_dict = {item["id"]:{} for item in llm_data}
    
    for item in llm_data:
        qid = item["id"]
        QA_dict[qid]["question"] = item["question"]
        QA_dict[qid]["llm_answer"] = item["answer"]

    for item in gt_data:
        qid = item["id"]
      #   import ipdb
      #   ipdb.set_trace()
        if qid in QA_dict.keys():
            QA_dict[qid]["gt_answer"] = item["conversations"][1]["value"]

    compares = []
    correct_ans = 0
    for ix, (qid, item) in enumerate(QA_dict.items()):
        question   = item["question"]
        llm_answer = item["llm_answer"]
        gt_answer  = item["gt_answer"]
        prompt = create_prompt(question, llm_answer, gt_answer)

        try:
            compare = get_result(prompt=prompt)

            answer, reason = extract_answer(compare)
            compare_data = {"id": qid, "answer": answer, "reason":reason}
            compares.append(compare_data)
            with open(out_file, 'w') as f:
                json.dump(compares, f, indent=4)
            
            if answer.lower()== 'yes':
                correct_ans = correct_ans + 1
                print(f"#correct \n answer:{answer},\n reason:{reason}")
            else:
                print(f"#wrong \n answer:{answer},\n reason:{reason}")
            
        except:
            print("break", item)
            continue
        print(f"[step {ix}, correct {correct_ans}, total {len(QA_dict)}, rate {correct_ans/len(QA_dict)}")  
        compares.append(compare)
    
    return compares



if __name__ == "__main__":
      
      #################################################
      #                4dor count                      #
      #################################################

      # # intern fintuned: [step 200, correct 75, total 200, rate 0.375]
      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results_eval/4dor_count_instruct_0711_test_compare.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # # intern origin: step 199, correct 18, total 200, rate 0.09
      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results_eval/4dor_count_instruct_0711_test_compare.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # llava 7b fintuned: [step 199, correct 111, total 200, rate 0.555]
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results_eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # ## llava 7b origin: [step 199, correct 44, total 200, rate 0.22]
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results-eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      ## llava 13b fintuned: [step 199, correct 125, total 200, rate 0.625]
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results_eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # ## llava 13b origin: [step 199, correct 16, total 200, rate 0.08]
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results-eval/4dor_count_instruct_0711_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      #################################################
      #                pwi count                      #
      #################################################

      # intern fintuned: step 199, correct 60, total 200, rate 0.3
      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/results_eval/pwiseg_count_instruct_0712_test_compare.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # # intern origin: step 199, correct 22, total 200, rate 0.11
      # llm_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/InternLM-XComposer/output/finetune_0712_pwi+4dor_epoch2/internlm-xcomposer2-vl-7b/origin_results_eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # llava 7b fintuned: step 198, correct 140, total 200, rate 0.7
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-task-lora-2024-07-14-08/results_eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # ## llava 7b origin:  step 199, correct 12, total 200, rate 0.06
      # llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
      # out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-7b-orign-results-eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
      # compares = main(llm_file, gt_file, out_file)

      # # # llava 13b fintuned: step 199, correct 142, total 200, rate 0.71
    #   llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
    #   gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
    #   out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-task-lora-2024-07-14-07/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
    #   compares = main(llm_file, gt_file, out_file)

      ## llava 13b origin: [step 199, correct 142, total 200, rate 0.71]
    #   llm_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
    #   gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
    #   out_file = '/mnt1/wjl/LLaVA/checkpoints/llava-v1.5-13b-origin-results-eval/pwiseg_count_instruct_0712_test_results.json'  # 替换为你的 LLM 预测文件路径
    #   compares = main(llm_file, gt_file, out_file)
    
      ## LLaVA-NeXT
      llm_file = '/mnt1/lyc/llava_finetune/eval_output/results_pwiseg_ori/preds_count.json'  # 替换为你的 LLM 预测文件路径
      gt_file = '/mnt1/wjl/InternLM-XComposer/output/GT/pwiseg_count_instruct_0712_test.json'      # 替换为你的 GT 文件路径
      out_file = '/mnt1/lyc/llava_finetune/eval_output/pwiseg_count_eval_llama3_llava.json'  # 替换为你的 LLM 预测文件路径
      
      
    #   llm_file = '/mnt1/lyc/llava_finetune/eval_output/results_4dor_ori/preds_count.json'  # 替换为你的 LLM 预测文件路径
    #   gt_file = '/mnt1/lyc/llava_finetune/data_json/4dor_count_instruct_0711_test.json'      # 替换为你的 GT 文件路径
    #   out_file = '/mnt1/lyc/llava_finetune/eval_output/4dor_count_eval_llama3_llava_ori.json'  # 替换为你的 LLM 预测文件路径
      
      compares = main(llm_file, gt_file, out_file)