File size: 8,641 Bytes
754ac61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157

import json

# with open('/home/aiops/wangzh/data/RGBD-benchmark/annotations.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/blink/BLINK/Counting/output.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/blink/BLINK/Relative_Depth/output.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/blink/BLINK/Spatial_Relation/output.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-13b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/blink/BLINK/Visual_Correspondence/output.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-13b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/blink/BLINK/Multi-view_Reasoning/output.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/blink/BLINK/Object_Localization/output.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-13b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/blink/BLINK/Jigsaw/output.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
with open('/home/aiops/wangzh/data/CV-Bench/test3d.jsonl', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/CV-Bench/test3d.jsonl', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:

#
# with open('/home/aiops/wangzh/data/blink/BLINK/Spatial_Relation/output.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/RGBD-benchmark/out_doors/all.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/RGBD-benchmark/out_doors/all.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/gpt4o-outdoor.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/scanner/indoor-new/all.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/gemini-indoor.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/scanner/indoor-new/all.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/gemini-indoor.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/realworldqa/updated.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:

# with open('/home/aiops/wangzh/data/RGBD-benchmark/out_doors/object_orientation.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/RGBD-benchmark/out_doors/relative_depth.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/RGBD-benchmark/out_doors/relative_size.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/RGBD-benchmark/out_doors/relative_spatial_position.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/scanner/indoor/none_spatial.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/scanner/indoor/orientation.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/scanner/indoor/relative_depth.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/scanner/indoor/relative_size.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:
# with open('/home/aiops/wangzh/data/scanner/indoor/spatial_relation.json', 'r') as reader1, open('/home/aiops/wangzh/llava-spat/llava-v1.5-7b-final-neg-lora_answers.txt', 'r') as reader2:

    # reader1 = json.load(reader1)

    correct = 0
    total = 0
    for index, (line1, line2) in enumerate(zip(reader1, reader2), 1):
        total += 1

        answer = line2.strip()
        # ground_truth = line1['answer']
        ground_truth = json.loads(line1.strip())['answer']
        # length = len(ground_truth)
        flag = False
        # choices = json.loads(line1.strip())['choices']

        # if ground_truth in answer:
        #     correct += 1
        # import pdb;pdb.set_trace()
        # if ('A' in answer) and ('B' in answer) and ('C' in answer) and ('D' in answer):
        #     print('missed',index)
        #     continue
        # if ground_truth == '(A)':
        #     if 'left' in answer:
        #         correct += 1
        #         print("yes",index)
        # elif ground_truth == '(B)':
        #     if 'right' in answer:
        #         correct += 1 
        #         print("yes",index)
        # if ground_truth == '(A)':
        #     if 'second' in answer:
        #         correct += 1
        #         print("yes",index)
        # elif ground_truth == '(B)':
        #     if 'third' in answer:
        #         correct += 1 
        #         print("yes",index)
        # count_sed = answer.count('sed')
        # count_tird = answer.count('tird')
        # if ground_truth == 0:
        #     if ('A' in answer) :
        #         correct += 1
        #         print("yes",index)

        # elif ground_truth == 1:
        #     if ('B' in answer) :
        #         correct += 1 
        #         print("yes",index)

        
        # elif ground_truth == 2:
        #     if ('C' in answer) :
        #         correct += 1 
        #         print("yes",index)

        
        # elif ground_truth == 3:
        #     if ('D' in answer) :
        #         correct += 1 
        #         print("yes",index)
        # if ground_truth == answer:
        #     correct += 1 
        #     print("yes",index)
        
        if ground_truth == '(A)':
            if 'A' in answer :
                correct += 1
                print("yes",index)
        elif ground_truth == '(B)':
            if 'B' in answer:
                correct += 1 
                print("yes",index)
        elif ground_truth == '(C)':
            if 'C' in answer:
                correct += 1 
                print("yes",index)
        elif ground_truth == '(D)':
            if 'D' in answer:
                correct += 1 
                print("yes",index)
        # if ground_truth == '(A)':
        #     if choices[2] in answer:
        #         correct += 1
        #         print("yes",index)
        # elif ground_truth == '(B)':
        #     if choices[6] in answer:
        #         correct += 1 
        #         print("yes",index)
        # elif ground_truth == '(C)':
        #     if choices[10] in answer:
        #         correct += 1 
        #         print("yes",index)
        # elif ground_truth == '(D)':
        #     if choices[14] in answer:
        #         correct += 1 
        #         print("yes",index)
        
    print("correct =", correct)
    print("total =", total)
    print("acc =",correct/total)



    # correct = 0
    # total = 0
    # for index, (line1, line2) in enumerate(zip(reader1, reader2), 1):
    #     total += 1
    #     answer = line2.strip()
    #     ground_truth = json.loads(line1.strip())['answer'].strip(".")[0]
    #     length = len(ground_truth)
    #     flag = False
    #     import pdb;pdb.set_trace()
    #     if length == 1 and ground_truth.isalpha():
    #         flag = True
    #         answer = answer.split(".")[0]
    #     elif length == 2 or length == 3:
    #         flag = True
    #         answer = answer.split(",")[0]

    #     if flag:
    #         if answer.lower() == ground_truth.lower():
    #             correct += 1
    #     else:
    #         print("->", index)
    # print("correct =", correct)
    # print("total =", total)