File size: 2,723 Bytes
fdf190d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
import re
import json
from tqdm import tqdm
import os
#names = os.listdir("processed_data")
f = open("/home/aiscuser/fhw/data/llama_instruct_final.json", "r+")
fw = open("/home/aiscuser/fhw/data/llama_instruct_selected.json", 'w+')
"""
lines = []
for name in names:
if "llama_python_scored" in name:
f = open(f"processed_data/{name}", 'r+')
lines.extend(f.readlines())
"""
lines = f.readlines()
for line in tqdm(lines):
d = json.loads(line)
instruction = d["instruction"]
judgement = d["quality_judgement"]
extracted = re.findall(r"\[\[(\d*\.\d+|\d+)/10\]\]", judgement, re.S)
if len(extracted) > 0:
d["score"] = float(extracted[-1])
fw.write(json.dumps(d)+"\n")
continue
extracted = re.findall(r"\[\[(\d*\.\d+|\d+)\]\]", judgement, re.S)
if len(extracted) > 0:
d["score"] = float(extracted[-1])
fw.write(json.dumps(d)+"\n")
continue
extracted = re.findall(r"\*\*Score: \[(\d*\.\d+|\d+)/10\]\*\*", judgement, re.S)
if len(extracted) > 0:
d["score"] = float(extracted[-1])
fw.write(json.dumps(d)+"\n")
continue
extracted = re.findall(r"\*\*Score: \[(\d*\.\d+|\d+)\]\*\*", judgement, re.S)
if len(extracted) > 0:
d["score"] = float(extracted[-1])
fw.write(json.dumps(d)+"\n")
continue
extracted = re.findall(r"\*\*Score: (\d*\.\d+|\d+)/10\*\*", judgement, re.S)
if len(extracted) > 0:
d["score"] = float(extracted[-1])
fw.write(json.dumps(d)+"\n")
continue
extracted = re.findall(r"\*\*Score: (\d*\.\d+|\d+)\*\*", judgement, re.S)
if len(extracted) > 0:
d["score"] = float(extracted[-1])
fw.write(json.dumps(d)+"\n")
continue
extracted = re.findall(r"\*\*Score:\*\* (\d*\.\d+|\d+)/10", judgement, re.S)
if len(extracted) > 0:
d["score"] = float(extracted[-1])
fw.write(json.dumps(d)+"\n")
continue
extracted = re.findall(r"\*\*Score:\*\* (\d*\.\d+|\d+)", judgement, re.S)
if len(extracted) > 0:
d["score"] = float(extracted[-1])
fw.write(json.dumps(d)+"\n")
continue
extracted = re.findall(r"Score(.*?)", judgement, re.S)
if len(extracted) > 0:
judgement = extracted[-1]
extracted = re.findall(r"\d*\.\d+|\d+", judgement, re.S)
if len(extracted) > 0:
d["score"] = float(extracted[-1])
fw.write(json.dumps(d)+"\n")
continue
extracted = re.findall(r"\d*\.\d+|\d+", judgement, re.S)
if len(extracted) > 0:
d["score"] = float(extracted[0])
fw.write(json.dumps(d)+"\n")
continue
#print("###########################################")
|