<filename>src/experiment/score.py
import argparse
import json
import sys
from fever.scorer import fever_score
from prettytable import PrettyTable

parser = argparse.ArgumentParser()
parser.add_argument("--predicted_labels",type=str)
parser.add_argument("--predicted_evidence",type=str, default=None)
parser.add_argument("--actual",type=str)

args = parser.parse_args()

predicted_labels =[]
predicted_evidence = []
actual = []

flatten = lambda l: [item for sublist in l for item in sublist]

with open(args.predicted_labels,"r") as predictions_file:
    for line in predictions_file:
        predicted_labels.append(json.loads(line)["predicted_label"])

with open(args.actual, "r") as actual_file:
    for line in actual_file:
        actual.append(json.loads(line))

with open(args.predicted_evidence, "r") as predictions_file:
    for line in predictions_file:
        line = json.loads(line)
        if "predicted_sentences" in line:
            predicted_evidence.append(line["predicted_sentences"])
        else:
            predicted_evidence.append([[e[2],e[3]] for e in flatten(line["evidence"])])


predictions = []


for ev,label in zip(predicted_evidence,predicted_labels):
    predictions.append({"predicted_evidence":ev,"predicted_label":label})

score,acc,precision,recall,f1 = fever_score(predictions,actual)

tab = PrettyTable()
tab.field_names = ["FEVER Score", "Label Accuracy", "Evidence Precision", "Evidence Recall", "Evidence F1"]
tab.add_row((round(score,4),round(acc,4),round(precision,4),round(recall,4),round(f1,4)))

print(tab)
<filename>src/experiment/score.py
import argparse
import json
import sys
from fever.scorer import fever_score
from prettytable import PrettyTable

parser = argparse.ArgumentParser()
parser.add_argument("--predicted_labels",type=str)
parser.add_argument("--predicted_evidence",type=str, default=None)
parser.add_argument("--actual",type=str)

args = parser.parse_args()

predicted_labels =[]
predicted_evidence = []
actual = []

flatten = lambda l: [item for sublist in l for item in sublist]

with open(args.predicted_labels,"r") as predictions_file:
    for line in predictions_file:
        predicted_labels.append(json.loads(line)["predicted_label"])

with open(args.actual, "r") as actual_file:
    for line in actual_file:
        actual.append(json.loads(line))

with open(args.predicted_evidence, "r") as predictions_file:
    for line in predictions_file:
        line = json.loads(line)
        if "predicted_sentences" in line:
            predicted_evidence.append(line["predicted_sentences"])
        else:
            predicted_evidence.append([[e[2],e[3]] for e in flatten(line["evidence"])])


predictions = []


for ev,label in zip(predicted_evidence,predicted_labels):
    predictions.append({"predicted_evidence":ev,"predicted_label":label})

score,acc,precision,recall,f1 = fever_score(predictions,actual)

tab = PrettyTable()
tab.field_names = ["FEVER Score", "Label Accuracy", "Evidence Precision", "Evidence Recall", "Evidence F1"]
tab.add_row((round(score,4),round(acc,4),round(precision,4),round(recall,4),round(f1,4)))

print(tab)