import argparse
import pickle
import random
import copy
import pandas as pd
import numpy as np
from collections import Counter
import os
from data_preprocessor import DataPreprocessor
def prepare_pretraining_files(data_processor, options):
# steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
val_file = open(options.val_file_path, "w")
val_info = open(options.val_info_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
# if options.workspace_name == section:
if "ratio_proportion_change3" == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
# step_names_token = [step for step in prob_groups['Step Name'] if str(step) != 'nan']
# print(step_names_token)
# writtenTrain = False
# writtenTest = False
student_groups.sort_values(by="Time", inplace=True)
# prob_list = list(pd.unique(student_groups["Problem Name"]))
# print(len(prob_list), prob_list)
# first_prob_list = prob_list[:3]
# last_prob_list = prob_list[-3:]
# print(len(first_prob_list), first_prob_list)
# print(len(last_prob_list), last_prob_list)
# final_prob_list = first_prob_list + last_prob_list
# print(len(prob_list), len(final_prob_list), final_prob_list)
for prob, prob_groups in student_groups.groupby("Problem Name"):
# For first 3 and last 3 only
# if not prob in final_prob_list:
# continue
# print(prob)
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups["Step Name"]))
unique_steps_len = len(set([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not(s in options.opt_step2)]))
if unique_steps_len < 4:
continue
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 1800:
time_stamps_list.add(time_stamps[i+1])
# progress = ""
step_names_token = []
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
for index, row in prob_groups[['Time', 'Step Name', 'CF (Etalon)', 'Outcome', 'Help Level','CF (Workspace Progress Status)']].iterrows():
step = row["Step Name"]
progress = row["CF (Workspace Progress Status)"]
etalon = row["CF (Etalon)"]
if not pd.isna(step):
if step in options.opt_step1 and not means_and_extremes:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
# if row["Time"] in time_stamps_list:
auto_complete = True
# print(row)
continue
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
unique_steps_len = len(set([s for s in step_names_token if not (s in options.opt_step1) and not(s in options.opt_step2)]))
# 4 and more in sequence
if step_names_token and unique_steps_len > 4:
# and len(step_names_token) > 3
# For information
# indices = [str(i) for i in prob_groups.index]
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
proba = random.random()
# if prob in first_prob_list:
if proba <= 0.8:
# writtenTrain = True
train_file.write("\t".join(step_names_token))
train_file.write("\n")
# train_info.write(",".join([str(progress),str(prob), str(student), str(len(step_names_token)),
# "\t".join(map(str, outcome)), "\t".join(map(str, help_level))]))
# progress, problem name, student id, auto_complete, total steps length, er or me, outcome seq, help_level seq, encoding in steps length
train_info.write(",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
f"{1 if means_and_extremes else 0}", "\t".join(map(str, outcome)),
"\t".join(map(str, help_level)), "\t".join(map(str, where_opt))]))
train_info.write("\n")
elif proba > 0.9:
# elif prob in last_prob_list:
# writtenTest = True
test_file.write("\t".join(step_names_token))
test_file.write("\n")
# test_info.write(",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
# "\t".join(map(str, outcome)), "\t".join(map(str, help_level))]))
# progress, problem name, student id, total steps length, er or me, outcome seq, help_level seq, encoding in steps length
test_info.write(",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
f"{1 if means_and_extremes else 0}", "\t".join(map(str, outcome)),
"\t".join(map(str, help_level)), "\t".join(map(str, where_opt))]))
test_info.write("\n")
else:
val_file.write("\t".join(step_names_token))
val_file.write("\n")
# test_info.write(",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
# "\t".join(map(str, outcome)), "\t".join(map(str, help_level))]))
# progress, problem name, student id, total steps length, er or me, outcome seq, help_level seq, encoding in steps length
val_info.write(",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
f"{1 if means_and_extremes else 0}", "\t".join(map(str, outcome)),
"\t".join(map(str, help_level)), "\t".join(map(str, where_opt))]))
val_info.write("\n")
# Indicates actions of next student
# Indicates next problem
# if writtenTrain:
# train_file.write("\n")
# train_info.write("\n")
# if writtenTest:
# test_file.write("\n")
# test_info.write("\n")
# if not writtenTrain and not writtenTest:
# print(f"Student {student} is not involved in workspace : {options.workspace_name}.")
train_file.close()
train_info.close()
val_file.close()
val_info.close()
test_file.close()
test_info.close()
def prepare_school_pretraining_files(data_processor, options):
# steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator(sep=",")
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
val_file = open(options.val_file_path, "w")
val_info = open(options.val_info_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
for chunk_data in chunk_iterator:
for school, school_group in chunk_data.groupby('CF (Anon School Id)'):
if not options.school or school in options.school:
for class_id, class_group in school_group.groupby('CF (Anon Class Id)'):
for student, student_groups in class_group.groupby("Anon Student Id"):
student_groups.sort_values(by="Time")
# prob_list = list(pd.unique(student_groups["Problem Name"]))
for prob, prob_groups in student_groups.groupby("Problem Name"):
# For first 3 and last 3 only
# if not prob in final_prob_list:
# continue
# print(prob)
step_names_token = []
means_and_extremes = False
for index, row in prob_groups[['Time', 'Step Name', 'Action', 'Attempt At Step', 'CF (Is Autofilled)', 'CF (Workspace Progress Status)', 'CF (Etalon)', 'CF (Problem Scenario Tags)']].iterrows():
progress = row["CF (Workspace Progress Status)"]
action = row["Action"]
attempt = row["Attempt At Step"]
autofilled = row["CF (Is Autofilled)"]
step = row["Step Name"]
scenario = row['CF (Problem Scenario Tags)']
if not pd.isna(step):
if step in options.opt_step1 and not means_and_extremes:
etalon = row["CF (Etalon)"]
if not pd.isna(etalon):
etalon = etalon.strip('{}')
key, value = etalon.split('=')
etalon = value
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
except Exception as e:
pass
if not autofilled:
new_step = f"{step}:{action}:{attempt}"
step_names_token.append(new_step)
if step_names_token:
where_opt = []
step1 = False
step2 = False
strategy_data = False
for step_oh in step_names_token:
step = step_oh.split(":")
if len(step) == 3:
step = step[0]
else:
step = ":".join(step[:2])
# print(f"changed {step_oh} = ? {step}")
if step == options.opt_step1[0]:
where_opt.append("_1")
step1 = True
elif step == options.opt_step2[0]:
where_opt.append("_2")
step2 = True
elif step in options.opt_step1[1:]:
where_opt.append("1")
if step1:
strategy_data = True
elif step in options.opt_step2[1:]:
where_opt.append("2")
if step2:
strategy_data = True
else:
where_opt.append("0")
if strategy_data and step_names_token[-1].split(":")[-2] != "Done":
strategy_data = False
if strategy_data:
proba = random.random()
step_names_tokens = [":".join(s.split(":")[:-2]) for s in step_names_token]
step_names_token = []
for s in step_names_tokens:
if s != "nan":
if not step_names_token or s != step_names_token[-1]:
step_names_token.append(s)
# if prob in first_prob_list:
if proba <= 0.8:
train_file.write("\t".join(step_names_token))
train_file.write("\n")
# school, class, student id, progress, problem name, scenario, prefered ER or ME, total steps length, encoding in steps length
train_info.write(",".join([str(school), str(class_id), str(student), str(progress), str(prob), str(scenario), f"{1 if means_and_extremes else 0}", str(len(step_names_token)), "\t".join(map(str, where_opt))]))
train_info.write("\n")
elif proba > 0.9:
# elif prob in last_prob_list:
test_file.write("\t".join(step_names_token))
test_file.write("\n")
# school, class, student id, progress, problem name, scenario, prefered ER or ME, total steps length, encoding in steps length
test_info.write(",".join([str(school), str(class_id), str(student), str(progress), str(prob), str(scenario), f"{1 if means_and_extremes else 0}", str(len(step_names_token)), "\t".join(map(str, where_opt))]))
test_info.write("\n")
else:
val_file.write("\t".join(step_names_token))
val_file.write("\n")
# school, class, student id, progress, problem name, scenario, prefered ER or ME, total steps length, encoding in steps length
val_info.write(",".join([str(school), str(class_id), str(student), str(progress), str(prob), str(scenario), f"{1 if means_and_extremes else 0}", str(len(step_names_token)), "\t".join(map(str, where_opt))]))
val_info.write("\n")
# break
# break
# break
# break
# break
train_file.close()
train_info.close()
val_file.close()
val_info.close()
test_file.close()
test_info.close()
def prepare_school_coded_pretraining_files(data_processor, options):
# steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator(sep=",")
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
val_file = open(options.val_file_path, "w")
val_info = open(options.val_info_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
for chunk_data in chunk_iterator:
for school, school_group in chunk_data.groupby('CF (Anon School Id)'):
if not options.school or school in options.school:
print(f"{school} : {school_group.shape}")
school_group = school_group[(school_group['CF (Is StepByStep)'] == False) &
(school_group['CF (Encounter)'] == 0) &
(school_group['CF (Is Review Mode)'] == -1) ]
print(f"{school} : {school_group.shape}")
# for class_id, class_group in school_groups.groupby('CF (Anon Class Id)'):
for student, student_groups in school_group.groupby("Anon Student Id"):
student_groups.sort_values(by="Time", inplace=True)
# At least 3 last problems are selected
prob_list= list(student_groups[student_groups["CF (Workspace Progress Status)"]=="GRADUATED"]["Problem Name"])
prob_list = prob_list[-int(len(prob_list)/2):]
for prob, prob_groups in student_groups.groupby("Problem Name"):
if not prob in prob_list:
continue
progress = list(pd.unique(prob_groups["CF (Workspace Progress Status)"]))[0]
if progress != "GRADUATED":
continue
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups[prob_groups["CF (Is Autofilled)"] == False]["Step Name"]))
unique_steps_len = len([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not (s in options.opt_step2)])
if unique_steps_len < 4:
continue
class_id = list(pd.unique(prob_groups["CF (Anon Class Id)"]))
step_names_token = []
original_steps_actions_attempts_help_levels_outcomes = []
original_steps = []
means_and_extremes = False
for index, row in prob_groups[['Step Name', 'Action', 'Attempt At Step', 'CF (Is Autofilled)',
'Outcome', 'Help Level', 'CF (Workspace Progress Status)',
'CF (Etalon)', 'CF (Problem Scenario Tags)']].iterrows():
step = row["Step Name"]
action = row["Action"] # ['Attempt', 'Hint Request', 'Hint Level Change', 'Done']
attempt = row["Attempt At Step"] # number
outcome = row["Outcome"] # ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
help_level = row["Help Level"] # number
# progress = row["CF (Workspace Progress Status)"]
scenario = row['CF (Problem Scenario Tags)']
if not pd.isna(step):
if step in options.opt_step1 and not means_and_extremes:
etalon = row["CF (Etalon)"]
if not pd.isna(etalon):
etalon = etalon.strip('{}')
key, value = etalon.split('=')
etalon = value
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
except Exception as e:
pass
if row['CF (Is Autofilled)'] == True:
continue
prev = step_names_token[-1] if step_names_token else ""
prev_step = step_names_token[-1].split("-")[0] if step_names_token else ""
if not step_names_token or step != prev_step:
if step in options.opt_step1 or step in options.opt_step2:
new_step = step
else:
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
step_names_token.append(new_step)
else:
if not (step in options.opt_step1 or step in options.opt_step2):
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if prev < new_step:
step_names_token[-1] = new_step
original_steps_actions_attempts_help_levels_outcomes.append(f"{step}-{action}-{attempt}-{help_level}-{outcome}")
original_steps.append(step)
unique_steps_len = len([s for s in original_steps if not (s in options.opt_step1) and not(s in options.opt_step2)])
if step_names_token and unique_steps_len > 4:
proba = random.random()
# if prob in first_prob_list:
if proba <= 0.8:
train_file.write("\t".join(step_names_token))
train_file.write("\n")
# school, class, student id, progress, problem name, scenario,
# prefered ER or ME, total steps length,
# original seq-action-attempt-help_level-outcome
train_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
train_info.write("\n")
elif proba > 0.9:
# elif prob in last_prob_list:
test_file.write("\t".join(step_names_token))
test_file.write("\n")
# school, class, student id, progress, problem name, scenario,
# prefered ER or ME, total steps length,
# original seq-action-attempt-help_level-outcome
test_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
test_info.write("\n")
else:
val_file.write("\t".join(step_names_token))
val_file.write("\n")
# school, class, student id, progress, problem name, scenario,
# prefered ER or ME, total steps length,
# original seq-action-attempt-help_level-outcome
val_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
val_info.write("\n")
# break
# break
# break
# break
# break
train_file.close()
train_info.close()
val_file.close()
val_info.close()
test_file.close()
test_info.close()
def prepare_school_attention_files(data_processor, options):
# steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator(sep=",")
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
val_file = open(options.val_file_path, "w")
val_info = open(options.val_info_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
for chunk_data in chunk_iterator:
for school, school_group in chunk_data.groupby('CF (Anon School Id)'):
if not options.school or school in options.school:
for class_id, class_group in school_group.groupby('CF (Anon Class Id)'):
for student, student_groups in class_group.groupby("Anon Student Id"):
student_groups.sort_values(by="Time")
# prob_list = list(pd.unique(student_groups["Problem Name"]))
# if len(prob_list) > 0 :
# first_fews = int(len(prob_list)/2)
# last_fews = len(prob_list) - first_fews
# first_prob_list = prob_list[:first_fews]
# last_prob_list = prob_list[-last_fews:]
# final_prob_list = first_prob_list + last_prob_list
for prob, prob_groups in student_groups.groupby("Problem Name"):
step_names_token = []
means_and_extremes = False
for index, row in prob_groups[['Time', 'Step Name', 'Action', 'Attempt At Step', 'CF (Is Autofilled)', 'CF (Workspace Progress Status)', 'CF (Etalon)', 'CF (Problem Scenario Tags)']].iterrows():
progress = row["CF (Workspace Progress Status)"]
action = row["Action"]
attempt = row["Attempt At Step"]
autofilled = row["CF (Is Autofilled)"]
step = row["Step Name"]
scenario = row['CF (Problem Scenario Tags)']
if not pd.isna(step):
if step in options.opt_step1 and not means_and_extremes:
etalon = row["CF (Etalon)"]
if not pd.isna(etalon):
etalon = etalon.strip('{}')
key, value = etalon.split('=')
etalon = value
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
except Exception as e:
pass
if not autofilled:
new_step = f"{step}:{action}:{attempt}"
step_names_token.append(new_step)
if step_names_token:
where_opt = []
step1 = False
step2 = False
strategy_data = False
for step_oh in step_names_token:
step = step_oh.split(":")
if len(step) == 3:
step = step[0]
else:
step = ":".join(step[:2])
# print(f"changed {step_oh} = ? {step}")
if step == options.opt_step1[0]:
where_opt.append("_1")
step1 = True
elif step == options.opt_step2[0]:
where_opt.append("_2")
step2 = True
elif step in options.opt_step1[1:]:
where_opt.append("1")
if step1:
strategy_data = True
elif step in options.opt_step2[1:]:
where_opt.append("2")
if step2:
strategy_data = True
else:
where_opt.append("0")
if strategy_data and step_names_token[-1].split(":")[-2] != "Done":
strategy_data = False
if strategy_data:
# proba = random.random()
step_names_tokens = [":".join(s.split(":")[:-2]) for s in step_names_token]
step_names_token = []
for s in step_names_tokens:
if s != "nan":
if not step_names_token or s != step_names_token[-1]:
step_names_token.append(s)
# if prob in first_prob_list:
if progress == "GRADUATED":# and means_and_extremes:# and prob in first_prob_list:
train_file.write("\t".join(step_names_token))
train_file.write("\n")
# school, class, student id, progress, problem name, scenario, prefered ER or ME, total steps length, encoding in steps length
train_info.write(",".join([str(school), str(class_id), str(student), str(progress), str(prob), str(scenario), f"{1 if means_and_extremes else 0}", str(len(step_names_token)), "\t".join(map(str, where_opt))]))
train_info.write("\n")
elif progress == "PROMOTED":# and means_and_extremes:# and prob in last_prob_list:
# elif prob in last_prob_list:
test_file.write("\t".join(step_names_token))
test_file.write("\n")
# school, class, student id, progress, problem name, scenario, prefered ER or ME, total steps length, encoding in steps length
test_info.write(",".join([str(school), str(class_id), str(student), str(progress), str(prob), str(scenario), f"{1 if means_and_extremes else 0}", str(len(step_names_token)), "\t".join(map(str, where_opt))]))
test_info.write("\n")
# else:
# val_file.write("\t".join(step_names_token))
# val_file.write("\n")
# # school, class, student id, progress, problem name, scenario, prefered ER or ME, total steps length, encoding in steps length
# val_info.write(",".join([str(school), str(class_id), str(student), str(progress), str(prob), str(scenario), f"{1 if means_and_extremes else 0}", str(len(step_names_token)), "\t".join(map(str, where_opt))]))
# val_info.write("\n")
# break
# break
# break
# break
# break
train_file.close()
train_info.close()
val_file.close()
val_info.close()
test_file.close()
test_info.close()
def prepare_finetuning_10per_files(data_processor, options):
'''
Used for L@S paper.
Only two strategies were defined as:
0: non-opt strategy
1: opt used strategy
'''
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
# val_file = open(options.val_file_path, "w")
# val_info = open(options.val_info_path, "w")
# val_label = open(options.val_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
overall_data = []
overall_labels = []
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
if "ratio_proportion_change3" == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
student_groups.sort_values(by="Time", inplace=True)
for prob, prob_groups in student_groups.groupby("Problem Name"):
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups["Step Name"]))
unique_steps_len = len(set([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not(s in options.opt_step2)]))
if unique_steps_len < 4:
continue
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 1800:
time_stamps_list.add(time_stamps[i+1])
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
auto_complete = True
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
unique_steps_len = len(set([s for s in step_names_token if not (s in options.opt_step1) and not(s in options.opt_step2)]))
# 4 and more in sequence
if step_names_token and unique_steps_len > 4:
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
label_opt = "0"
if options.opt_step1:
any_opt1 = any(any(opt in step for step in step_names_token) for opt in options.opt_step1[1:])
if any_opt1:
label_opt = "1"
if options.opt_step2:
any_opt2 = any(any(opt in step for step in step_names_token) for opt in options.opt_step2[1:])
if any_opt2:
label_opt = "1"
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
info = ",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt)),
f"{1 if means_and_extremes else 0}"])
overall_data.append(["\t".join(step_names_token), info])
overall_labels.append(label_opt)
# overall_data.append('')
# overall_labels.append('')
overall_labels = np.array(overall_labels)
indices_of_zeros = list(np.where(overall_labels == '0')[0])
indices_of_ones = list(np.where(overall_labels == '1')[0])
train_len = int(len(overall_labels) * 0.10)
sample_size = int(train_len/2)
print(f"sample_size: {sample_size}")
sampled_instances = random.sample(indices_of_zeros, sample_size)
sampled_instances.extend(random.sample(indices_of_ones, sample_size))
balanced_test = min(len(indices_of_zeros), len(indices_of_ones))
test_sampled_instances = random.sample(indices_of_zeros, balanced_test)
test_sampled_instances.extend(random.sample(indices_of_ones, balanced_test))
# writtenTrain = False
# writtenTest = False
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
steps_seq = all_data[0]
info = all_data[1]
if index in sampled_instances:
train_file.write(steps_seq)
train_file.write("\n")
train_info.write(info)
train_info.write("\n")
train_label.write(label)
train_label.write("\n")
elif index in test_sampled_instances:
# proba = random.random()
# if proba <0.5:
test_file.write(steps_seq)
test_file.write("\n")
test_info.write(info)
test_info.write("\n")
test_label.write(label)
test_label.write("\n")
# else:
# val_file.write(steps_seq)
# val_file.write("\n")
# val_info.write(info)
# val_info.write("\n")
# val_label.write(label)
# val_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
# val_file.close()
# val_info.close()
# val_label.close()
test_file.close()
test_info.close()
test_label.close()
def prepare_finetuning_IS_FS_files(data_processor, options):
'''
Used for L@S paper. This function gathers first three problems of each student.
Only two strategies were defined as:
0: non-opt strategy
1: opt used strategy
train: IS
test: FS
'''
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
# val_file = open(options.val_file_path, "w")
# val_info = open(options.val_info_path, "w")
# val_label = open(options.val_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
if "ratio_proportion_change3" == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
student_groups.sort_values(by="Time", inplace=True)
prob_list = list(pd.unique(student_groups["Problem Name"]))
if len(prob_list) < 3:
continue
selected = 3 #1. int(len(prob_list)/2)
#2. 3 & <6
#3. 3 & <3
first_prob_list = prob_list[:selected]
last_prob_list = prob_list[-selected:]
for prob, prob_groups in student_groups.groupby("Problem Name"):
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups["Step Name"]))
unique_steps_len = len(set([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not(s in options.opt_step2)]))
if unique_steps_len < 4:
continue
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 1800:
time_stamps_list.add(time_stamps[i+1])
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
auto_complete = True
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
unique_steps_len = len(set([s for s in step_names_token if not (s in options.opt_step1) and not(s in options.opt_step2)]))
# 4 and more in sequence
if step_names_token and unique_steps_len > 4:
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
label_opt = "0"
if options.opt_step1:
any_opt1 = any(any(opt in step for step in step_names_token) for opt in options.opt_step1[1:])
if any_opt1:
label_opt = "1"
if options.opt_step2:
any_opt2 = any(any(opt in step for step in step_names_token) for opt in options.opt_step2[1:])
if any_opt2:
label_opt = "1"
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
info = ",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt)),
f"{1 if means_and_extremes else 0}"])
if prob in first_prob_list:
train_file.write("\t".join(step_names_token))
train_file.write("\n")
train_info.write(info)
train_info.write("\n")
train_label.write(label_opt)
train_label.write("\n")
elif prob in last_prob_list:
test_file.write("\t".join(step_names_token))
test_file.write("\n")
test_info.write(info)
test_info.write("\n")
test_label.write(label_opt)
test_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
# val_file.close()
# val_info.close()
# val_label.close()
test_file.close()
test_info.close()
test_label.close()
def prepare_finetuning_IS_files_old(data_processor, opts):
'''
Used for L@S paper. This function gathers first three problems of each student.
Only two strategies were defined as:
0: non-opt strategy
1: opt used strategy
'''
options = copy.deepcopy(opts)
for k,v in vars(opts).items():
if k.startswith("train") or k.startswith("test"):
if v:
f_path = v.split("/")
f_path = f_path[0]+"/"+f_path[1]+"/IS/"+f_path[2]
setattr(options, f"{k}", f_path)
print(f"options.{k} : {getattr(options, f'{k}')}")
steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
trainr_label = open(options.trainr_label_path, "w")
train_gt_label = open(options.train_gt_label_path, "w")
# test_file = open(options.test_file_path, "w")
# test_info = open(options.test_info_path, "w")
# test_label = open(options.test_label_path, "w")
# testr_label = open(options.testr_label_path, "w")
# test_gt_label = open(options.test_gt_label_path, "w")
overall_data = []
overall_labels = []
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
if options.workspace_name == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
student_groups.sort_values(by="Time")
prob_list = list(pd.unique(student_groups["Problem Name"]))
if len(prob_list) < 3:
continue
first_prob_list = prob_list[:3]
# last_prob_list = prob_list[-3:]
# # print(len(first_prob_list), len(last_prob_list))
# final_prob_list = first_prob_list + last_prob_list
# print(len(prob_list), len(final_prob_list)) #, final_prob_list)
for prob, prob_groups in student_groups.groupby("Problem Name"):
# For first 3 and last 3 only
if not prob in first_prob_list:
continue
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 2000:
time_stamps_list.add(time_stamps[i+1])
progress = ""
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
finals = len(options.final_step)
totals = 0
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
# if row["Time"] in time_stamps_list:
auto_complete = True
# print(row)
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
if finals == 0:
totals += 1
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
error_ind = ['BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
errors = 0
for step, out in zip(step_names_token, outcome):
if (finals and step in options.final_step) or totals > 0:
out = out.split(":")
if any(any(ind in o for o in out) for ind in error_ind):
errors +=1
if finals:
totals = finals
# 4 and more in sequence
if step_names_token: # and len(step_names_token) > 3
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
label_opt = "0"
if options.opt_step1:
any_opt1 = any(any(opt in step for step in step_names_token) for opt in options.opt_step1[1:])
if any_opt1:
label_opt = "1"
if options.opt_step2:
any_opt2 = any(any(opt in step for step in step_names_token) for opt in options.opt_step2[1:])
if any_opt2:
label_opt = "1"
correctness = 1 - errors/totals
strat_correct = "0"
if correctness > 0.75:
strat_correct = "1"
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
info = ",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt)), str(correctness)])
overall_data.append(["\t".join(step_names_token), strat_correct, info, f"{1 if means_and_extremes else 0}"])
overall_labels.append(label_opt)
overall_data.append('')
overall_labels.append('')
# overall_labels = np.array(overall_labels)
# indices_of_zeros = list(np.where(overall_labels == '0')[0])
# indices_of_ones = list(np.where(overall_labels == '1')[0])
# zeros_instances_size = int(1 * len(indices_of_zeros))
# ones_instances_size = int(1 * len(indices_of_ones))
# sample_size = min(zeros_instances_size, ones_instances_size)
# sampled_instances = random.sample(indices_of_zeros, sample_size)
# sampled_instances.extend(random.sample(indices_of_ones, sample_size))
writtenTrain = False
# writtenTest = False
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
if all_data:
steps_seq = all_data[0]
strat_correct = all_data[1]
info = all_data[2]
me_opt = all_data[3]
# if index in sampled_instances:
writtenTrain = True
train_file.write(steps_seq)
train_file.write("\n")
train_label.write(label)
train_label.write("\n")
trainr_label.write(strat_correct)
trainr_label.write("\n")
train_info.write(info)
train_info.write("\n")
train_gt_label.write(me_opt)
train_gt_label.write("\n")
# else:
# writtenTest = True
# test_file.write(steps_seq)
# test_file.write("\n")
# # progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
# test_label.write(label)
# test_label.write("\n")
# # testr_label.write(str(correctness))
# testr_label.write(strat_correct)
# testr_label.write("\n")
# test_info.write(info)
# test_info.write("\n")
# test_gt_label.write(me_opt)
# test_gt_label.write("\n")
else:
# Indicates actions of next student
# Indicates next problem
if writtenTrain:
writtenTrain = False
train_file.write("\n")
train_info.write("\n")
train_label.write("\n")
trainr_label.write("\n")
train_gt_label.write("\n")
# if writtenTest:
# writtenTest = False
# test_file.write("\n")
# test_info.write("\n")
# test_label.write("\n")
# testr_label.write("\n")
# test_gt_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
trainr_label.close()
train_gt_label.close()
# test_file.close()
# test_info.close()
# test_label.close()
# testr_label.close()
# test_gt_label.close()
def prepare_finetuning_FS_files_old(data_processor, opts):
'''
Used for L@S paper. This function gathers last three problems of each student.
Only two strategies were defined as:
0: non-opt strategy
1: opt used strategy
'''
options = copy.deepcopy(opts)
for k,v in vars(opts).items():
if k.startswith("train") or k.startswith("test"):
if v:
f_path = v.split("/")
f_path = f_path[0]+"/"+f_path[1]+"/FS/"+f_path[2]
setattr(options, f"{k}", f_path)
print(f"options.{k} : {getattr(options, f'{k}')}")
steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
trainr_label = open(options.trainr_label_path, "w")
train_gt_label = open(options.train_gt_label_path, "w")
# test_file = open(options.test_file_path, "w")
# test_info = open(options.test_info_path, "w")
# test_label = open(options.test_label_path, "w")
# testr_label = open(options.testr_label_path, "w")
# test_gt_label = open(options.test_gt_label_path, "w")
overall_data = []
overall_labels = []
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
if options.workspace_name == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
student_groups.sort_values(by="Time")
prob_list = list(pd.unique(student_groups["Problem Name"]))
if len(prob_list) < 3:
continue
# first_prob_list = prob_list[:3]
last_prob_list = prob_list[-3:]
# # print(len(first_prob_list), len(last_prob_list))
# final_prob_list = first_prob_list + last_prob_list
# print(len(prob_list), len(final_prob_list)) #, final_prob_list)
for prob, prob_groups in student_groups.groupby("Problem Name"):
# For first 3 and last 3 only
if not prob in last_prob_list:
continue
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 2000:
time_stamps_list.add(time_stamps[i+1])
progress = ""
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
finals = len(options.final_step)
totals = 0
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
# if row["Time"] in time_stamps_list:
auto_complete = True
# print(row)
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
if finals == 0:
totals += 1
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
error_ind = ['BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
errors = 0
for step, out in zip(step_names_token, outcome):
if (finals and step in options.final_step) or totals > 0:
out = out.split(":")
if any(any(ind in o for o in out) for ind in error_ind):
errors +=1
if finals:
totals = finals
# 4 and more in sequence
if step_names_token: # and len(step_names_token) > 3
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
label_opt = "0"
if options.opt_step1:
any_opt1 = any(any(opt in step for step in step_names_token) for opt in options.opt_step1[1:])
if any_opt1:
label_opt = "1"
if options.opt_step2:
any_opt2 = any(any(opt in step for step in step_names_token) for opt in options.opt_step2[1:])
if any_opt2:
label_opt = "1"
correctness = 1 - errors/totals
strat_correct = "0"
if correctness > 0.75:
strat_correct = "1"
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
info = ",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt)), str(correctness)])
overall_data.append(["\t".join(step_names_token), strat_correct, info, f"{1 if means_and_extremes else 0}"])
overall_labels.append(label_opt)
overall_data.append('')
overall_labels.append('')
# overall_labels = np.array(overall_labels)
# indices_of_zeros = list(np.where(overall_labels == '0')[0])
# indices_of_ones = list(np.where(overall_labels == '1')[0])
# zeros_instances_size = int(0.10 * len(indices_of_zeros))
# ones_instances_size = int(0.10 * len(indices_of_ones))
# sample_size = min(zeros_instances_size, ones_instances_size)
# sampled_instances = random.sample(indices_of_zeros, sample_size)
# sampled_instances.extend(random.sample(indices_of_ones, sample_size))
writtenTrain = False
# writtenTest = False
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
if all_data:
steps_seq = all_data[0]
strat_correct = all_data[1]
info = all_data[2]
me_opt = all_data[3]
# if index in sampled_instances:
writtenTrain = True
train_file.write(steps_seq)
train_file.write("\n")
train_label.write(label)
train_label.write("\n")
trainr_label.write(strat_correct)
trainr_label.write("\n")
train_info.write(info)
train_info.write("\n")
train_gt_label.write(me_opt)
train_gt_label.write("\n")
# else:
# writtenTest = True
# test_file.write(steps_seq)
# test_file.write("\n")
# # progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
# test_label.write(label)
# test_label.write("\n")
# # testr_label.write(str(correctness))
# testr_label.write(strat_correct)
# testr_label.write("\n")
# test_info.write(info)
# test_info.write("\n")
# test_gt_label.write(me_opt)
# test_gt_label.write("\n")
else:
# Indicates actions of next student
# Indicates next problem
if writtenTrain:
writtenTrain = False
train_file.write("\n")
train_info.write("\n")
train_label.write("\n")
trainr_label.write("\n")
train_gt_label.write("\n")
# if writtenTest:
# writtenTest = False
# test_file.write("\n")
# test_info.write("\n")
# test_label.write("\n")
# testr_label.write("\n")
# test_gt_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
trainr_label.close()
train_gt_label.close()
# test_file.close()
# test_info.close()
# test_label.close()
# testr_label.close()
# test_gt_label.close()
def prepare_finetuning_correctness_files(data_processor, options):
'''
Ongoing research. Student strategy learning/predicting.
FinalAnswer step
Correct: 1 , correctness of final strategy > 0.75
Incorrect: 0 , else < 0.75
'''
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
# val_file = open(options.val_file_path, "w")
# val_info = open(options.val_info_path, "w")
# val_label = open(options.val_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
overall_data = []
overall_labels = []
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
if "ratio_proportion_change3" == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
student_groups.sort_values(by="Time", inplace=True)
for prob, prob_groups in student_groups.groupby("Problem Name"):
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups["Step Name"]))
unique_steps_len = len(set([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not(s in options.opt_step2)]))
if unique_steps_len < 4:
continue
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 1800:
time_stamps_list.add(time_stamps[i+1])
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
final_correct = 0
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
auto_complete = True
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
if step == "FinalAnswer":
final_correct += 1
unique_steps_len = len(set([s for s in step_names_token if not (s in options.opt_step1) and not(s in options.opt_step2)]))
# 4 and more in sequence
if step_names_token and unique_steps_len > 4:
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
label_opt = "0"
if final_correct == 1:
label_opt = "1"
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
info = ",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt)),
f"{1 if means_and_extremes else 0}"])
overall_data.append(["\t".join(step_names_token), info])
overall_labels.append(label_opt)
# overall_data.append('')
# overall_labels.append('')
overall_labels = np.array(overall_labels)
indices_of_zeros = list(np.where(overall_labels == '0')[0])
indices_of_ones = list(np.where(overall_labels == '1')[0])
train_len = int(len(overall_labels) * 0.10)
sample_size = int(train_len/2)
print(f"sample_size: {sample_size}")
sampled_instances = random.sample(indices_of_zeros, sample_size)
sampled_instances.extend(random.sample(indices_of_ones, sample_size))
# writtenTrain = False
# writtenTest = False
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
steps_seq = all_data[0]
info = all_data[1]
if index in sampled_instances:
train_file.write(steps_seq)
train_file.write("\n")
train_info.write(info)
train_info.write("\n")
train_label.write(label)
train_label.write("\n")
else:
# proba = random.random()
# if proba <0.5:
test_file.write(steps_seq)
test_file.write("\n")
test_info.write(info)
test_info.write("\n")
test_label.write(label)
test_label.write("\n")
# else:
# val_file.write(steps_seq)
# val_file.write("\n")
# val_info.write(info)
# val_info.write("\n")
# val_label.write(label)
# val_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
# val_file.close()
# val_info.close()
# val_label.close()
test_file.close()
test_info.close()
test_label.close()
def prepare_finetuning_correctness_files_old(data_processor, opts):
'''
Ongoing research. Student strategy learning/predicting.
Correct, 1: correctness of final strategy > 0.75
Incorrect, 0: else < 0.75
'''
options = copy.deepcopy(opts)
for k,v in vars(opts).items():
if k.startswith("train") or k.startswith("test"):
if v:
f_path = v.split("/")
f_path = f_path[0]+"/"+f_path[1]+"/fa_correctness/"+f_path[2]
# f_path = f_path[0]+"/"+f_path[1]+"/check2/"+f_path[2]
setattr(options, f"{k}", f_path)
print(f"options.{k} : {getattr(options, f'{k}')}")
steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
# trainr_label = open(options.trainr_label_path, "w")
# train_gt_label = open(options.train_gt_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
# testr_label = open(options.testr_label_path, "w")
# test_gt_label = open(options.test_gt_label_path, "w")
ws = "_".join(options.workspace_name.split("_")[:-1])
print("Workspace: ", ws)
overall_data = []
overall_labels = []
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
if ws == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
student_groups.sort_values(by="Time")
prob_list = list(pd.unique(student_groups["Problem Name"]))
# if len(prob_list) < 3:
# continue
# first_prob_list = prob_list[:3]
# last_prob_list = prob_list[-3:]
# # print(len(first_prob_list), len(last_prob_list))
# final_prob_list = first_prob_list + last_prob_list
# print(len(prob_list), len(final_prob_list)) #, final_prob_list)
for prob, prob_groups in student_groups.groupby("Problem Name"):
# For first 3 and last 3 only
# if not prob in last_prob_list:
# continue
# print(options.final_step in list(prob_groups["Step Name"]))
# if not (options.final_step in list(prob_groups["Step Name"])):
# continue
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 2000:
time_stamps_list.add(time_stamps[i+1])
progress = ""
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
# finals = len(options.final_step)
totals = 0
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
# if row["Time"] in time_stamps_list:
auto_complete = True
# print(row)
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
# if finals == 0:
# totals += 1
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
error_ind = ['BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
errors = 0
for step, out in zip(step_names_token, outcome):
if (step in options.final_step):# or totals > 0:
out = out.split(":")
totals = len(out)
# print(totals)
for ind in error_ind:
if ind in out:
errors +=1
# if finals:
# totals = finals
# 4 and more in sequence
if step_names_token and totals>0: # and len(step_names_token) > 3
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
label_opt = "0"
if options.opt_step1:
all_opt1 = all(any(opt in step for step in step_names_token) for opt in options.opt_step1)
any_opt1 = any(any(opt in step for step in step_names_token) for opt in options.opt_step1[1:])
if any_opt1:
label_opt = "2"
if all_opt1:
label_opt = "1"
if options.opt_step2:
all_opt2 = all(any(opt in step for step in step_names_token) for opt in options.opt_step2)
any_opt2 = any(any(opt in step for step in step_names_token) for opt in options.opt_step2[1:])
if any_opt2:
label_opt = "4"
if all_opt2:
label_opt = "3"
if any_opt1 and any_opt2:
label_opt = "5"
if any_opt1 and all_opt2:
label_opt = "6"
if all_opt1 and any_opt2:
label_opt = "7"
if all_opt1 and all_opt2:
label_opt = "8"
correctness = 1 - errors/totals
strat_correct = "0"
if correctness > 0.75:
strat_correct = "1"
# if not means_and_extremes and label_opt == "2":
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
info = ",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt)), str(correctness), f"{1 if means_and_extremes else 0}"])
overall_data.append(["\t".join(step_names_token), label_opt, info])
overall_labels.append(strat_correct)
overall_data.append('')
overall_labels.append('')
overall_labels = np.array(overall_labels, dtype=str)
indices_of_zeros = list(np.where(overall_labels == '0')[0])
indices_of_ones = list(np.where(overall_labels == '1')[0])
per = 0.20
zeros_instances_size = int(per * len(indices_of_zeros))
ones_instances_size = int(per * len(indices_of_ones))
sample_size = min(zeros_instances_size, ones_instances_size)
sampled_instances = random.sample(indices_of_zeros, sample_size)
sampled_instances.extend(random.sample(indices_of_ones, sample_size))
writtenTrain = False
writtenTest = False
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
if all_data:
steps_seq = all_data[0]
label_opt = all_data[1]
info = all_data[2]
# me_opt = all_data[3]
if index in sampled_instances:
writtenTrain = True
train_file.write(steps_seq)
train_file.write("\n")
train_label.write(label)
train_label.write("\n")
# trainr_label.write(label_opt)
# trainr_label.write("\n")
train_info.write(info)
train_info.write("\n")
# train_gt_label.write(me_opt)
# train_gt_label.write("\n")
else:
writtenTest = True
test_file.write(steps_seq)
test_file.write("\n")
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
test_label.write(label)
test_label.write("\n")
# testr_label.write(str(correctness))
# testr_label.write(label_opt)
# testr_label.write("\n")
test_info.write(info)
test_info.write("\n")
# test_gt_label.write(me_opt)
# test_gt_label.write("\n")
else:
# Indicates actions of next student
# Indicates next problem
if writtenTrain:
writtenTrain = False
train_file.write("\n")
train_info.write("\n")
train_label.write("\n")
# trainr_label.write("\n")
# train_gt_label.write("\n")
if writtenTest:
writtenTest = False
test_file.write("\n")
test_info.write("\n")
test_label.write("\n")
# testr_label.write("\n")
# test_gt_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
# trainr_label.close()
# train_gt_label.close()
test_file.close()
test_info.close()
test_label.close()
# testr_label.close()
# test_gt_label.close()
def prepare_finetuning_correctness_aaai_files(data_processor, opts):
'''
Ongoing research. Student strategy learning/predicting.
Correct, 1: correctness of final strategy > 0.75
Incorrect, 0: else < 0.75
'''
options = copy.deepcopy(opts)
for k,v in vars(opts).items():
if k.startswith("train") or k.startswith("test") or k.startswith("val"):
if v:
f_path = v.split("/")
# f_path = f_path[0]+"/"+f_path[1]+"/correctness/"+f_path[2]
f_path = f_path[0]+"/"+f_path[1]+"/aaai/"+f_path[2]
setattr(options, f"{k}", f_path)
print(f"options.{k} : {getattr(options, f'{k}')}")
# steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
val_file = open(options.val_file_path, "w")
val_info = open(options.val_info_path, "w")
val_label = open(options.val_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
high_performer = pickle.load(open(f"{options.workspace_name}/aaai/change3_high_performers.pkl", "rb"))
mid_performer = pickle.load(open(f"{options.workspace_name}/aaai/change3_mid_performers.pkl", "rb"))
low_performer = pickle.load(open(f"{options.workspace_name}/aaai/change3_low_performers.pkl", "rb"))
prob_sel_list = pickle.load(open(f"{options.workspace_name}/aaai/change3_problem_list.pkl", "rb"))
ws = "_".join(options.workspace_name.split("_")[:-1])
print(ws, len(high_performer), len(mid_performer), len(low_performer), len(prob_sel_list))
overall_data = []
overall_labels = []
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
# if options.workspace_name == section:
if ws == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
if student in high_performer or student in mid_performer or student in low_performer:
student_groups.sort_values(by="Time")
prob_list = list(pd.unique(student_groups["Problem Name"]))
for prob, prob_groups in student_groups.groupby("Problem Name"):
# For first 3 and last 3 only
if not prob in prob_sel_list:
continue
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 2000:
time_stamps_list.add(time_stamps[i+1])
progress = ""
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
totals = 0
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
# if row["Time"] in time_stamps_list:
auto_complete = True
# print(row)
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
error_ind = ['BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
errors = 0
for step, out in zip(step_names_token, outcome):
if (step in options.final_step):
out = out.split(":")
totals = len(out)
# print(totals)
for ind in error_ind:
if ind in out:
errors +=1
# 4 and more in sequence
if step_names_token and totals>0: # and len(step_names_token) > 3
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
correctness = 1 - errors/totals
strat_correct = "0"
if correctness > 0.75:
strat_correct = "1"
# if not means_and_extremes and label_opt == "2":
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
info = ",".join([str(progress), str(correctness), f"{1 if means_and_extremes else 0}",str(prob), str(student), str(auto_complete), str(len(step_names_token)),"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt))])
overall_data.append(["\t".join(step_names_token), info])
overall_labels.append(strat_correct)
# overall_data.append('')
# overall_labels.append('')
overall_labels = np.array(overall_labels)
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
if all_data:
steps_seq = all_data[0]
info = all_data[1]
student = info.split(",")[4]
if student in high_performer:
train_file.write(steps_seq)
train_file.write("\n")
train_label.write(label)
train_label.write("\n")
train_info.write(info)
train_info.write("\n")
elif student in mid_performer:
val_file.write(steps_seq)
val_file.write("\n")
val_label.write(label)
val_label.write("\n")
val_info.write(info)
val_info.write("\n")
elif student in low_performer:
test_file.write(steps_seq)
test_file.write("\n")
test_label.write(label)
test_label.write("\n")
test_info.write(info)
test_info.write("\n")
train_file.close()
train_info.close()
train_label.close()
val_file.close()
val_info.close()
val_label.close()
test_file.close()
test_info.close()
test_label.close()
def prepare_finetuning_SL_files(data_processor, opts):
'''
Ongoing research. Student strategy learning/predicting.
We have defined 9 strategy as:
Notation; Label
UU; 0
CU; 1
PU; 2
UC; 3
UP; 4
PP; 5
PC; 6
CP; 7
CC; 8
'''
options = copy.deepcopy(opts)
for k,v in vars(opts).items():
if k.startswith("train") or k.startswith("test"):
if v:
f_path = v.split("/")
f_path = f_path[0]+"/"+f_path[1]+"/SL/"+f_path[2]
setattr(options, f"{k}", f_path)
print(f"options.{k} : {getattr(options, f'{k}')}")
steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
trainr_label = open(options.trainr_label_path, "w")
train_gt_label = open(options.train_gt_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
testr_label = open(options.testr_label_path, "w")
test_gt_label = open(options.test_gt_label_path, "w")
overall_data = []
overall_labels = []
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
if options.workspace_name == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
student_groups.sort_values(by="Time")
prob_list = list(pd.unique(student_groups["Problem Name"]))
# if len(prob_list) < 3:
# continue
# first_prob_list = prob_list[:3]
# last_prob_list = prob_list[-3:]
# # print(len(first_prob_list), len(last_prob_list))
# final_prob_list = first_prob_list + last_prob_list
# print(len(prob_list), len(final_prob_list)) #, final_prob_list)
for prob, prob_groups in student_groups.groupby("Problem Name"):
# For first 3 and last 3 only
# if not prob in last_prob_list:
# continue
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 2000:
time_stamps_list.add(time_stamps[i+1])
progress = ""
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
finals = len(options.final_step)
totals = 0
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
# if row["Time"] in time_stamps_list:
auto_complete = True
# print(row)
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
if finals == 0:
totals += 1
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
error_ind = ['BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
errors = 0
for step, out in zip(step_names_token, outcome):
if (finals and step in options.final_step) or totals > 0:
out = out.split(":")
if any(any(ind in o for o in out) for ind in error_ind):
errors +=1
if finals:
totals = finals
# 4 and more in sequence
if step_names_token: # and len(step_names_token) > 3
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
label_opt = "0"
if options.opt_step1:
all_opt1 = all(any(opt in step for step in step_names_token) for opt in options.opt_step1)
any_opt1 = any(any(opt in step for step in step_names_token) for opt in options.opt_step1[1:])
if any_opt1:
label_opt = "2"
if all_opt1:
label_opt = "1"
if options.opt_step2:
all_opt2 = all(any(opt in step for step in step_names_token) for opt in options.opt_step2)
any_opt2 = any(any(opt in step for step in step_names_token) for opt in options.opt_step2[1:])
if any_opt2:
label_opt = "4"
if all_opt2:
label_opt = "3"
if any_opt1 and any_opt2:
label_opt = "5"
if any_opt1 and all_opt2:
label_opt = "6"
if all_opt1 and any_opt2:
label_opt = "7"
if all_opt1 and all_opt2:
label_opt = "8"
correctness = 1 - errors/totals
strat_correct = "0"
if correctness > 0.75:
strat_correct = "1"
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
info = ",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt)), str(correctness)])
overall_data.append(["\t".join(step_names_token), strat_correct, info, f"{1 if means_and_extremes else 0}"])
overall_labels.append(label_opt)
overall_data.append('')
overall_labels.append('')
overall_labels = np.array(overall_labels)
indices_of_zeros = list(np.where(overall_labels == '0')[0])
indices_of_ones = list(np.where(overall_labels == '1')[0])
indices_of_twos = list(np.where(overall_labels == '2')[0])
indices_of_threes = list(np.where(overall_labels == '3')[0])
indices_of_fours = list(np.where(overall_labels == '4')[0])
indices_of_fives = list(np.where(overall_labels == '5')[0])
indices_of_sixes = list(np.where(overall_labels == '6')[0])
indices_of_sevens = list(np.where(overall_labels == '7')[0])
indices_of_eights = list(np.where(overall_labels == '8')[0])
per = 0.20
zeros_instances_size = int(per * len(indices_of_zeros))
ones_instances_size = int(per * len(indices_of_ones))
twos_instances_size = int(per * len(indices_of_twos))
threes_instances_size = int(per * len(indices_of_threes))
fours_instances_size = int(per * len(indices_of_fours))
fives_instances_size = int(per * len(indices_of_fives))
sixes_instances_size = int(per * len(indices_of_sixes))
sevens_instances_size = int(per * len(indices_of_sevens))
eights_instances_size = int(per * len(indices_of_eights))
sample_size = min(zeros_instances_size, ones_instances_size, twos_instances_size, threes_instances_size, fours_instances_size, fives_instances_size, sixes_instances_size, sevens_instances_size, eights_instances_size)
print(f"Sample size.... {sample_size}")
sampled_instances = random.sample(indices_of_zeros, sample_size)
sampled_instances.extend(random.sample(indices_of_ones, sample_size))
sampled_instances.extend(random.sample(indices_of_twos, sample_size))
sampled_instances.extend(random.sample(indices_of_threes, sample_size))
sampled_instances.extend(random.sample(indices_of_fours, sample_size))
sampled_instances.extend(random.sample(indices_of_fives, sample_size))
sampled_instances.extend(random.sample(indices_of_sixes, sample_size))
sampled_instances.extend(random.sample(indices_of_sevens, sample_size))
sampled_instances.extend(random.sample(indices_of_eights, sample_size))
writtenTrain = False
writtenTest = False
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
if all_data:
steps_seq = all_data[0]
strat_correct = all_data[1]
info = all_data[2]
me_opt = all_data[3]
if index in sampled_instances:
writtenTrain = True
train_file.write(steps_seq)
train_file.write("\n")
train_label.write(label)
train_label.write("\n")
trainr_label.write(strat_correct)
trainr_label.write("\n")
train_info.write(info)
train_info.write("\n")
train_gt_label.write(me_opt)
train_gt_label.write("\n")
else:
writtenTest = True
test_file.write(steps_seq)
test_file.write("\n")
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
test_label.write(label)
test_label.write("\n")
# testr_label.write(str(correctness))
testr_label.write(strat_correct)
testr_label.write("\n")
test_info.write(info)
test_info.write("\n")
test_gt_label.write(me_opt)
test_gt_label.write("\n")
else:
# Indicates actions of next student
# Indicates next problem
if writtenTrain:
writtenTrain = False
train_file.write("\n")
train_info.write("\n")
train_label.write("\n")
trainr_label.write("\n")
train_gt_label.write("\n")
if writtenTest:
writtenTest = False
test_file.write("\n")
test_info.write("\n")
test_label.write("\n")
testr_label.write("\n")
test_gt_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
trainr_label.close()
train_gt_label.close()
test_file.close()
test_info.close()
test_label.close()
testr_label.close()
test_gt_label.close()
def prepare_finetuning_effectiveness_files(data_processor, opts):
'''
Ongoing research. Student strategy learning/predicting.
We have defined 9 strategy as:
Notation; Label
UU; 0
CU; 1
PU; 2
UC; 3
UP; 4
PP; 5
PC; 6
CP; 7
CC; 8
if UU and CU and PU and gt = ER and correct, a positive instance
if UU and UC and UP and gt = ME and correct, a positive instance
else a strategy PP, PC, CP, CC and gt = ER/ME or incorrect, a negative instance
'''
options = copy.deepcopy(opts)
for k,v in vars(opts).items():
if k.startswith("train") or k.startswith("test"):
if v:
f_path = v.split("/")
f_path = f_path[0]+"/"+f_path[1]+"/effectiveness/"+f_path[2]
setattr(options, f"{k}", f_path)
print(f"options.{k} : {getattr(options, f'{k}')}")
steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
trainr_label = open(options.trainr_label_path, "w")
train_gt_label = open(options.train_gt_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
testr_label = open(options.testr_label_path, "w")
test_gt_label = open(options.test_gt_label_path, "w")
overall_data = []
overall_labels = []
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
if options.workspace_name == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
student_groups.sort_values(by="Time")
prob_list = list(pd.unique(student_groups["Problem Name"]))
# if len(prob_list) < 3:
# continue
# first_prob_list = prob_list[:3]
# last_prob_list = prob_list[-3:]
# # print(len(first_prob_list), len(last_prob_list))
# final_prob_list = first_prob_list + last_prob_list
# print(len(prob_list), len(final_prob_list)) #, final_prob_list)
for prob, prob_groups in student_groups.groupby("Problem Name"):
# For first 3 and last 3 only
# if not prob in last_prob_list:
# continue
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 2000:
time_stamps_list.add(time_stamps[i+1])
progress = ""
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
finals = len(options.final_step)
totals = 0
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
# if row["Time"] in time_stamps_list:
auto_complete = True
# print(row)
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
if finals == 0:
totals += 1
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
error_ind = ['BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
errors = 0
for step, out in zip(step_names_token, outcome):
if (finals and step in options.final_step) or totals > 0:
out = out.split(":")
if any(any(ind in o for o in out) for ind in error_ind):
errors +=1
if finals:
totals = finals
# 4 and more in sequence
if step_names_token: # and len(step_names_token) > 3
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
label_opt = "0"
if options.opt_step1:
all_opt1 = all(any(opt in step for step in step_names_token) for opt in options.opt_step1)
any_opt1 = any(any(opt in step for step in step_names_token) for opt in options.opt_step1[1:])
if any_opt1:
label_opt = "2"
if all_opt1:
label_opt = "1"
if options.opt_step2:
all_opt2 = all(any(opt in step for step in step_names_token) for opt in options.opt_step2)
any_opt2 = any(any(opt in step for step in step_names_token) for opt in options.opt_step2[1:])
if any_opt2:
label_opt = "4"
if all_opt2:
label_opt = "3"
if any_opt1 and any_opt2:
label_opt = "5"
if any_opt1 and all_opt2:
label_opt = "6"
if all_opt1 and any_opt2:
label_opt = "7"
if all_opt1 and all_opt2:
label_opt = "8"
correctness = 1 - errors/totals
strat_correct = "0"
if correctness > 0.75:
strat_correct = "1"
label_effectiveness = "0"
if label_opt in ["0", "1", "2"] and not means_and_extremes and strat_correct == "1":
label_effectiveness = "1"
elif label_opt in ["0", "3", "4"] and means_and_extremes and strat_correct == "1":
label_effectiveness = "1"
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
info = ",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt)), str(correctness), label_opt, f"{1 if means_and_extremes else 0}"])
overall_data.append(["\t".join(step_names_token), strat_correct, info, f"{1 if means_and_extremes else 0}"])
overall_labels.append(label_effectiveness)
overall_data.append('')
overall_labels.append('')
overall_labels = np.array(overall_labels)
indices_of_zeros = list(np.where(overall_labels == '0')[0])
indices_of_ones = list(np.where(overall_labels == '1')[0])
per = 0.20
zeros_instances_size = int(per * len(indices_of_zeros))
ones_instances_size = int(per * len(indices_of_ones))
sample_size = min(zeros_instances_size, ones_instances_size)
sampled_instances = random.sample(indices_of_zeros, sample_size)
sampled_instances.extend(random.sample(indices_of_ones, sample_size))
writtenTrain = False
writtenTest = False
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
if all_data:
steps_seq = all_data[0]
strat_correct = all_data[1]
info = all_data[2]
me_opt = all_data[3]
if index in sampled_instances:
writtenTrain = True
train_file.write(steps_seq)
train_file.write("\n")
train_label.write(label)
train_label.write("\n")
trainr_label.write(strat_correct)
trainr_label.write("\n")
train_info.write(info)
train_info.write("\n")
train_gt_label.write(me_opt)
train_gt_label.write("\n")
else:
writtenTest = True
test_file.write(steps_seq)
test_file.write("\n")
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
test_label.write(label)
test_label.write("\n")
# testr_label.write(str(correctness))
testr_label.write(strat_correct)
testr_label.write("\n")
test_info.write(info)
test_info.write("\n")
test_gt_label.write(me_opt)
test_gt_label.write("\n")
else:
# Indicates actions of next student
# Indicates next problem
if writtenTrain:
writtenTrain = False
train_file.write("\n")
train_info.write("\n")
train_label.write("\n")
trainr_label.write("\n")
train_gt_label.write("\n")
if writtenTest:
writtenTest = False
test_file.write("\n")
test_info.write("\n")
test_label.write("\n")
testr_label.write("\n")
test_gt_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
trainr_label.close()
train_gt_label.close()
test_file.close()
test_info.close()
test_label.close()
testr_label.close()
test_gt_label.close()
def prepare_attn_test_files(data_processor, opts):
options = copy.deepcopy(opts)
if options.code:
new_folder = f"{options.workspace_name}/{options.code}"
if not os.path.exists(new_folder):
os.makedirs(new_folder)
for k,v in vars(opts).items():
if k.startswith("train") or k.startswith("test"):
if v:
f_path = (f"/{options.code}/").join(v.split("/"))
setattr(options, f"{k}", f_path)
print(f"options.{k} : {getattr(options, f'{k}')}")
steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
if options.code != "full":
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
if options.workspace_name == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
student_groups.sort_values(by="Time")
prob_list = list(pd.unique(student_groups["Problem Name"]))
for prob, prob_groups in student_groups.groupby("Problem Name"):
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 2000:
time_stamps_list.add(time_stamps[i+1])
progress = ""
outcome = []
help_level = []
auto_complete = False
means_and_extremes = False
finals = len(options.final_step)
totals = 0
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
# if row["Time"] in time_stamps_list:
auto_complete = True
# print(row)
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
if finals == 0:
totals += 1
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
error_ind = ['BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
errors = 0
for step, out in zip(step_names_token, outcome):
if (finals and step in options.final_step) or totals > 0:
out = out.split(":")
if any(any(ind in o for o in out) for ind in error_ind):
errors +=1
if finals:
totals = finals
# 4 and more in sequence
if step_names_token: # and len(step_names_token) > 3
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
label_opt = "0"
if options.opt_step1:
all_opt1 = all(any(opt in step for step in step_names_token) for opt in options.opt_step1)
any_opt1 = any(any(opt in step for step in step_names_token) for opt in options.opt_step1[1:])
if any_opt1:
label_opt = "2"
if all_opt1:
label_opt = "1"
if options.opt_step2:
all_opt2 = all(any(opt in step for step in step_names_token) for opt in options.opt_step2)
any_opt2 = any(any(opt in step for step in step_names_token) for opt in options.opt_step2[1:])
if any_opt2:
label_opt = "4"
if all_opt2:
label_opt = "3"
if any_opt1 and any_opt2:
label_opt = "5"
if any_opt1 and all_opt2:
label_opt = "6"
if all_opt1 and any_opt2:
label_opt = "7"
if all_opt1 and all_opt2:
label_opt = "8"
correctness = 1 - errors/totals
opt_correct = "0"
if correctness > 0.75:
opt_correct = "1"
proba = random.random()
# if proba <= 0.1:
# if not means_and_extremes:
# if prob in first_prob_list:
if options.code == "full" or (options.code == "gt" and not means_and_extremes) or (options.code == "correct" and opt_correct == "1") or (options.code == "progress" and progress == "GRADUATED"):
if label_opt == "0":
continue
train_file.write("\t".join(step_names_token))
train_file.write("\n")
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
train_info.write(",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt)),
str(correctness), f"{1 if means_and_extremes else 0}", label_opt]))
train_info.write("\n")
# if means_and_extremes:
# if prob in last_prob_list:
else:
test_file.write("\t".join(step_names_token))
test_file.write("\n")
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
test_info.write(",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt)),
str(correctness), f"{1 if means_and_extremes else 0}", label_opt]))
test_info.write("\n")
train_file.close()
train_info.close()
if options.code != "full":
test_file.close()
test_info.close()
def prepare_finetuning_future_files(data_processor, opts):
options = copy.deepcopy(opts)
for k,v in vars(opts).items():
if k.startswith("train") or k.startswith("test"):
if v:
f_path = ("/effectiveness/").join(v.split("/"))
setattr(options, f"{k}", f_path)
print(f"options.{k} : {getattr(options, f'{k}')}")
steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
chunk_iterator = data_processor.load_file_iterator()
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
trainr_label = open(options.trainr_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
testr_label = open(options.testr_label_path, "w")
for chunk_data in chunk_iterator:
for section, section_groups in chunk_data.groupby("Level (Workspace Id)"):
if options.workspace_name == section:
for student, student_groups in section_groups.groupby("Anon Student Id"):
writtenTrain = False
writtenTest = False
student_groups.sort_values(by="Time")
prob_list = list(pd.unique(student_groups["Problem Name"]))
# if len(prob_list) < 6:
# continue
# first_prob_list = prob_list[:3]
# last_prob_list = prob_list[-3:]
# # print(len(first_prob_list), len(last_prob_list))
# final_prob_list = first_prob_list + last_prob_list
# print(len(prob_list), len(final_prob_list)) #, final_prob_list)
for prob, prob_groups in student_groups.groupby("Problem Name"):
# For first 3 and last 3 only
# if not prob in final_prob_list:
# continue
step_names_token = []
time_stamps = list(prob_groups["Time"])
time_stamps_list = set()
for i in range(len(time_stamps)-1):
if (time_stamps[i+1] - time_stamps[i]) < 2000:
time_stamps_list.add(time_stamps[i+1])
progress = ""
outcome = []
help_level = []
auto_complete = False
errors = 0
totals = 0
means_and_extremes = False
for index, row in prob_groups[['Time', 'Step Name', 'Outcome', 'Help Level','CF (Workspace Progress Status)', 'CF (Etalon)']].iterrows():
step = row["Step Name"]
etalon = row["CF (Etalon)"]
progress = row["CF (Workspace Progress Status)"]
if not pd.isna(step):
if step in options.opt_step1:
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
# break
except Exception as e:
pass
if (step in options.opt_step1 or step in options.opt_step2) and row["Time"] in time_stamps_list:
# if row["Time"] in time_stamps_list:
auto_complete = True
# print(row)
continue
# if not step_names_token or step != step_names_token[-1]:
# step_names_token.append(step)
if not step_names_token or step != step_names_token[-1]:
step_names_token.append(step)
# ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
outcome.append(row['Outcome'])
help_level.append(str(row["Help Level"]))
totals += 1
else:
outcome[-1] = outcome[-1]+":"+row['Outcome']
help_level[-1] = help_level[-1]+":"+str(row['Help Level'])
error_ind = ['BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
for out in outcome:
out = out.split(":")
if any(any(ind in o for o in out) for ind in error_ind):
errors +=1
# 4 and more in sequence
if step_names_token: # and len(step_names_token) > 3
where_opt = []
for stp in step_names_token:
if stp in options.opt_step1:
where_opt.append("1")
elif stp in options.opt_step2:
where_opt.append("2")
else:
where_opt.append("0")
label_opt = "0"
if options.opt_step1:
all_opt1 = all(any(opt in step for step in step_names_token) for opt in options.opt_step1)
any_opt1 = any(any(opt in step for step in step_names_token) for opt in options.opt_step1[1:])
if any_opt1:
label_opt = "2"
if all_opt1:
label_opt = "1"
if options.opt_step2:
all_opt2 = all(any(opt in step for step in step_names_token) for opt in options.opt_step2)
any_opt2 = any(any(opt in step for step in step_names_token) for opt in options.opt_step2[1:])
if any_opt2:
label_opt = "4"
if all_opt2:
label_opt = "3"
if any_opt1 and any_opt2:
label_opt = "5"
if any_opt1 and all_opt2:
label_opt = "6"
if all_opt1 and any_opt2:
label_opt = "7"
if all_opt1 and all_opt2:
label_opt = "8"
correctness = 1 - errors/totals
opt_correct = "0"
if correctness < 0.25:
opt_correct = "0"
elif correctness < 0.5:
opt_correct = "1"
elif correctness < 0.75:
opt_correct = "2"
else:
opt_correct = "3"
proba = random.random()
# if proba <= 0.1:
if not means_and_extremes:
# if prob in first_prob_list:
writtenTrain = True
train_file.write("\t".join(step_names_token))
train_file.write("\n")
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
train_label.write(label_opt)
train_label.write("\n")
# trainr_label.write(str(correctness))
trainr_label.write(opt_correct)
trainr_label.write("\n")
train_info.write(",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt))]))
train_info.write("\n")
if means_and_extremes:
# if prob in last_prob_list:
# else:
writtenTest = True
test_file.write("\t".join(step_names_token))
test_file.write("\n")
# progress, problem name, student id, total steps length, outcome seq, help_level seq, encoding in steps length
test_label.write(label_opt)
test_label.write("\n")
# testr_label.write(str(correctness))
testr_label.write(opt_correct)
testr_label.write("\n")
test_info.write(",".join([str(progress),str(prob), str(student), str(auto_complete), str(len(step_names_token)),
"\t".join(map(str, outcome)), "\t".join(map(str, help_level)), "\t".join(map(str, where_opt))]))
test_info.write("\n")
# Indicates actions of next student
# Indicates next problem
if writtenTrain:
train_file.write("\n")
train_info.write("\n")
train_label.write("\n")
trainr_label.write("\n")
if writtenTest:
test_file.write("\n")
test_info.write("\n")
test_label.write("\n")
testr_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
trainr_label.close()
test_file.close()
test_info.close()
test_label.close()
testr_label.close()
def prepare_school_coded_finetuning_partial_seq_files(data_processor, options):
'''
Ongoing research.
FinalAnswer step correctness
Correct: 0 if attempt at step>1
1 if attempt at step==1
'''
chunk_iterator = data_processor.load_file_iterator(sep=",")
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
# val_file = open(options.val_file_path, "w")
# val_info = open(options.val_info_path, "w")
# val_label = open(options.val_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
overall_data = []
overall_labels = []
# kcs = pickle.load(open("dataset/CL_2223/ratio_proportion_change3_2223/unique_kcs_list.pkl", "rb"))
# kcs = [kc if not pd.isna(kc) for kc in kcs]
for chunk_data in chunk_iterator:
for school, school_group in chunk_data.groupby('CF (Anon School Id)'):
if not options.school or school in options.school:
print(f"{school} : {school_group.shape}")
school_group = school_group[(school_group['CF (Is StepByStep)'] == False) &
(school_group['CF (Encounter)'] == 0) &
(school_group['CF (Is Review Mode)'] == -1) ]
print(f"{school} : {school_group.shape}")
# for class_id, class_group in school_groups.groupby('CF (Anon Class Id)'):
for student, student_groups in school_group.groupby("Anon Student Id"):
student_groups.sort_values(by="Time", inplace=True)
# prob_list = list(pd.unique(student_groups["Problem Name"]))
for prob, prob_groups in student_groups.groupby("Problem Name"):
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups[prob_groups["CF (Is Autofilled)"] == False]["Step Name"]))
unique_steps_len = len([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not (s in options.opt_step2)])
if unique_steps_len < 4:
continue
unique_opt_steps_len = len([s for s in unique_steps if not pd.isna(s) and (s in options.opt_step1[1:] or s in options.opt_step2[1:])])
if unique_opt_steps_len < 2:
continue
class_id = list(pd.unique(prob_groups["CF (Anon Class Id)"]))
step_names_token = []
original_steps_actions_attempts_help_levels_outcomes = []
original_steps = []
means_and_extremes = False
correctness = "0"
opt_used = False
for index, row in prob_groups[['Step Name', 'Action', 'Attempt At Step', 'CF (Is Autofilled)',
'Outcome', 'Help Level', 'CF (Workspace Progress Status)',
'CF (Etalon)', 'CF (Problem Scenario Tags)']].iterrows():
step = row["Step Name"]
action = row["Action"] # ['Attempt', 'Hint Request', 'Hint Level Change', 'Done']
attempt = row["Attempt At Step"] # number
outcome = row["Outcome"] # ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
help_level = row["Help Level"] # number
progress = row["CF (Workspace Progress Status)"]
scenario = row['CF (Problem Scenario Tags)']
if not pd.isna(step):
if step in options.opt_step1 and not means_and_extremes:
etalon = row["CF (Etalon)"]
if not pd.isna(etalon):
etalon = etalon.strip('{}')
key, value = etalon.split('=')
etalon = value
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
except Exception as e:
pass
if row['CF (Is Autofilled)'] == True:
continue
prev = step_names_token[-1] if step_names_token else ""
prev_step = step_names_token[-1].split("-")[0] if step_names_token else ""
if not step_names_token or step != prev_step:
if step in options.opt_step1 or step in options.opt_step2:
new_step = step
opt_used = True
else:
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if step != "FinalAnswer":
step_names_token.append(new_step)
else:
step_names_token.append("FinalAnswer")
else:
if not (step in options.opt_step1 or step in options.opt_step2 or step == "FinalAnswer"):
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if prev < new_step:
step_names_token[-1] = new_step
if step == "FinalAnswer" and opt_used:
if attempt == 1 and outcome == "OK":
correctness = "1"
else:
correctness = "0"
original_steps_actions_attempts_help_levels_outcomes.append(f"{step}-{action}-{attempt}-{help_level}-{outcome}")
original_steps.append(step)
unique_steps_len = len([s for s in original_steps if not (s in options.opt_step1) and not(s in options.opt_step2)])
if step_names_token and unique_steps_len > 4:
info = ",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))])
overall_data.append(["\t".join(step_names_token), info])
overall_labels.append(correctness)
# proba = random.random()
# # if prob in first_prob_list:
# if proba <= 0.8:
# train_file.write("\t".join(step_names_token))
# train_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# train_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# train_info.write("\n")
# elif proba > 0.9:
# # elif prob in last_prob_list:
# test_file.write("\t".join(step_names_token))
# test_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# test_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# test_info.write("\n")
# else:
# val_file.write("\t".join(step_names_token))
# val_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# val_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# val_info.write("\n")
# break
# break
# break
# break
# break
overall_labels = np.array(overall_labels)
indices_of_zeros = list(np.where(overall_labels == '0')[0])
indices_of_ones = list(np.where(overall_labels == '1')[0])
train_len = int(len(overall_labels) * 0.10)
sample_size = int(train_len/2)
print(f"sample_size: {sample_size}")
sampled_instances = random.sample(indices_of_zeros, sample_size)
sampled_instances.extend(random.sample(indices_of_ones, sample_size))
indices_of_zeros = [i for i in indices_of_zeros if not i in sampled_instances ]
indices_of_ones = [i for i in indices_of_ones if not i in sampled_instances ]
balanced_test = min(len(indices_of_zeros), len(indices_of_ones))
print(f"balanced_test: {balanced_test}")
test_sampled_instances = random.sample(indices_of_zeros, balanced_test)
test_sampled_instances.extend(random.sample(indices_of_ones, balanced_test))
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
steps_seq = all_data[0]
info = all_data[1]
if index in sampled_instances:
train_file.write(steps_seq)
train_file.write("\n")
train_info.write(info)
train_info.write("\n")
train_label.write(label)
train_label.write("\n")
elif index in test_sampled_instances:
# proba = random.random()
# if proba <0.5:
test_file.write(steps_seq)
test_file.write("\n")
test_info.write(info)
test_info.write("\n")
test_label.write(label)
test_label.write("\n")
# else:
# val_file.write(steps_seq)
# val_file.write("\n")
# val_info.write(info)
# val_info.write("\n")
# val_label.write(label)
# val_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
# val_file.close()
# val_info.close()
# val_label.close()
test_file.close()
test_info.close()
test_label.close()
def prepare_school_coded_finetuning_opts_files(data_processor, options):
'''
Ongoing research.
Labels:
0 - Opt 1
1 - Opt 2
2 - Both Opt
'''
chunk_iterator = data_processor.load_file_iterator(sep=",")
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
# val_file = open(options.val_file_path, "w")
# val_info = open(options.val_info_path, "w")
# val_label = open(options.val_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
overall_data = []
overall_labels = []
# kcs = pickle.load(open("dataset/CL_2223/ratio_proportion_change3_2223/unique_kcs_list.pkl", "rb"))
# kcs = [kc if not pd.isna(kc) for kc in kcs]
for chunk_data in chunk_iterator:
for school, school_group in chunk_data.groupby('CF (Anon School Id)'):
if not options.school or school in options.school:
print(f"{school} : {school_group.shape}")
school_group = school_group[(school_group['CF (Is StepByStep)'] == False) &
(school_group['CF (Encounter)'] == 0) &
(school_group['CF (Is Review Mode)'] == -1) ]
print(f"{school} : {school_group.shape}")
# for class_id, class_group in school_groups.groupby('CF (Anon Class Id)'):
for student, student_groups in school_group.groupby("Anon Student Id"):
student_groups.sort_values(by="Time", inplace=True)
# prob_list = list(pd.unique(student_groups["Problem Name"]))
# prob_list= list(student_groups[student_groups["CF (Workspace Progress Status)"]=="GRADUATED"]["Problem Name"])
# prob_list = prob_list[-int(len(prob_list)/2):]
for prob, prob_groups in student_groups.groupby("Problem Name"):
# if not prob in prob_list:
# continue
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups[prob_groups["CF (Is Autofilled)"] == False]["Step Name"]))
unique_steps_len = len([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not (s in options.opt_step2)])
if unique_steps_len < 4:
continue
unique_opt_steps_len = len([s for s in unique_steps if not pd.isna(s) and (s in options.opt_step1[1:] or s in options.opt_step2[1:])])
if unique_opt_steps_len < 2:
continue
print(unique_steps, unique_opt_steps_len)
class_id = list(pd.unique(prob_groups["CF (Anon Class Id)"]))
step_names_token = []
original_steps_actions_attempts_help_levels_outcomes = []
original_steps = []
means_and_extremes = False
opt1_used = False
opt2_used = False
for index, row in prob_groups[['Step Name', 'Action', 'Attempt At Step', 'CF (Is Autofilled)',
'Outcome', 'Help Level', 'CF (Workspace Progress Status)',
'CF (Etalon)', 'CF (Problem Scenario Tags)']].iterrows():
step = row["Step Name"]
action = row["Action"] # ['Attempt', 'Hint Request', 'Hint Level Change', 'Done']
attempt = row["Attempt At Step"] # number
outcome = row["Outcome"] # ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
help_level = row["Help Level"] # number
progress = row["CF (Workspace Progress Status)"]
scenario = row['CF (Problem Scenario Tags)']
if not pd.isna(step):
if step in options.opt_step1 and not means_and_extremes:
etalon = row["CF (Etalon)"]
if not pd.isna(etalon):
etalon = etalon.strip('{}')
key, value = etalon.split('=')
etalon = value
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
except Exception as e:
pass
if row['CF (Is Autofilled)'] == True:
continue
prev = step_names_token[-1] if step_names_token else ""
prev_step = step_names_token[-1].split("-")[0] if step_names_token else ""
if not step_names_token or step != prev_step:
if step in options.opt_step1 or step in options.opt_step2:
new_step = step
if step in options.opt_step1[1:]:
opt1_used = True
elif step in options.opt_step2[2:]:
opt2_used = True
else:
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
step_names_token.append(new_step)
else:
if not (step in options.opt_step1 or step in options.opt_step2 or step == "FinalAnswer"):
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if prev < new_step:
step_names_token[-1] = new_step
original_steps_actions_attempts_help_levels_outcomes.append(f"{step}-{action}-{attempt}-{help_level}-{outcome}")
original_steps.append(step)
if (not opt1_used) and (not opt2_used):
continue
unique_steps_len = len([s for s in original_steps if not (s in options.opt_step1) and not(s in options.opt_step2)])
if step_names_token and unique_steps_len > 4:
info = ",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))])
overall_data.append(["\t".join(step_names_token), info])
label = None
if opt1_used and opt2_used:
label = "2"
if (not opt1_used) and opt2_used:
label = "1"
if opt1_used and (not opt2_used):
label = "0"
print(f"opt1_used: {opt1_used}, opt2_used: {opt2_used} label : {label}")
overall_labels.append(label)
# proba = random.random()
# # if prob in first_prob_list:
# if proba <= 0.8:
# train_file.write("\t".join(step_names_token))
# train_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# train_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# train_info.write("\n")
# elif proba > 0.9:
# # elif prob in last_prob_list:
# test_file.write("\t".join(step_names_token))
# test_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# test_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# test_info.write("\n")
# else:
# val_file.write("\t".join(step_names_token))
# val_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# val_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# val_info.write("\n")
# break
# break
# break
# break
# break
overall_labels = np.array(overall_labels)
indices_of_zeros = list(np.where(overall_labels == '0')[0])
indices_of_ones = list(np.where(overall_labels == '1')[0])
indices_of_twos = list(np.where(overall_labels == '2')[0])
train_len = int(len(overall_labels) * 0.10)
sample_size = int(train_len/3)
print(f"sample_size: {sample_size}")
sampled_instances = random.sample(indices_of_zeros, sample_size)
sampled_instances.extend(random.sample(indices_of_ones, sample_size))
sampled_instances.extend(random.sample(indices_of_twos, sample_size))
indices_of_zeros = [i for i in indices_of_zeros if not i in sampled_instances ]
indices_of_ones = [i for i in indices_of_ones if not i in sampled_instances ]
indices_of_twos = [i for i in indices_of_twos if not i in sampled_instances ]
balanced_test = min(len(indices_of_zeros), len(indices_of_ones), len(indices_of_twos))
print(f"balanced_test: {balanced_test}")
test_sampled_instances = random.sample(indices_of_zeros, balanced_test)
test_sampled_instances.extend(random.sample(indices_of_ones, balanced_test))
test_sampled_instances.extend(random.sample(indices_of_twos, balanced_test))
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
steps_seq = all_data[0]
info = all_data[1]
if index in sampled_instances:
train_file.write(steps_seq)
train_file.write("\n")
train_info.write(info)
train_info.write("\n")
train_label.write(label)
train_label.write("\n")
elif index in test_sampled_instances:
# proba = random.random()
# if proba <0.5:
test_file.write(steps_seq)
test_file.write("\n")
test_info.write(info)
test_info.write("\n")
test_label.write(label)
test_label.write("\n")
# else:
# val_file.write(steps_seq)
# val_file.write("\n")
# val_info.write(info)
# val_info.write("\n")
# val_label.write(label)
# val_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
# val_file.close()
# val_info.close()
# val_label.close()
test_file.close()
test_info.close()
test_label.close()
def prepare_school_coded_finetuning_opts_intentional_files(data_processor, options):
'''
Ongoing research.
Labels:
0 - Opt 1
1 - Opt 2
2 - Both Opt
'''
chunk_iterator = data_processor.load_file_iterator(sep=",")
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
val_file = open(options.val_file_path, "w")
val_info = open(options.val_info_path, "w")
val_label = open(options.val_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
# overall_data = []
# overall_labels = []
# kcs = pickle.load(open("dataset/CL_2223/ratio_proportion_change3_2223/unique_kcs_list.pkl", "rb"))
# kcs = [kc if not pd.isna(kc) for kc in kcs]
for chunk_data in chunk_iterator:
for school, school_group in chunk_data.groupby('CF (Anon School Id)'):
if not options.school or school in options.school:
print(f"{school} : {school_group.shape}")
school_group = school_group[(school_group['CF (Is StepByStep)'] == False) &
(school_group['CF (Encounter)'] == 0) &
(school_group['CF (Is Review Mode)'] == -1) ]
print(f"{school} : {school_group.shape}")
# for class_id, class_group in school_groups.groupby('CF (Anon Class Id)'):
for student, student_groups in school_group.groupby("Anon Student Id"):
student_groups.sort_values(by="Time", inplace=True)
# prob_list = list(pd.unique(student_groups["Problem Name"]))
prob_list= list(pd.unique(student_groups[student_groups["CF (Workspace Progress Status)"]=="GRADUATED"]["Problem Name"]))
# prob_list = prob_list[-int(len(prob_list)/2):]
if len(prob_list) == 0:
continue
for prob, prob_groups in student_groups.groupby("Problem Name"):
# if not prob in prob_list:
# continue
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups[prob_groups["CF (Is Autofilled)"] == False]["Step Name"]))
unique_steps_len = len([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not (s in options.opt_step2)])
if unique_steps_len < 4:
continue
unique_opt_steps_len = len([s for s in unique_steps if not pd.isna(s) and (s in options.opt_step1[1:] or s in options.opt_step2[1:])])
if unique_opt_steps_len < 2:
continue
# print(unique_steps, unique_opt_steps_len)
class_id = list(pd.unique(prob_groups["CF (Anon Class Id)"]))
step_names_token = []
original_steps_actions_attempts_help_levels_outcomes = []
original_steps = []
means_and_extremes = False
opt1_used = False
opt2_used = False
for index, row in prob_groups[['Step Name', 'Action', 'Attempt At Step', 'CF (Is Autofilled)',
'Outcome', 'Help Level', 'CF (Workspace Progress Status)',
'CF (Etalon)', 'CF (Problem Scenario Tags)']].iterrows():
step = row["Step Name"]
action = row["Action"] # ['Attempt', 'Hint Request', 'Hint Level Change', 'Done']
attempt = row["Attempt At Step"] # number
outcome = row["Outcome"] # ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
help_level = row["Help Level"] # number
progress = row["CF (Workspace Progress Status)"]
scenario = row['CF (Problem Scenario Tags)']
if not pd.isna(step):
if step in options.opt_step1 and not means_and_extremes:
etalon = row["CF (Etalon)"]
if not pd.isna(etalon):
etalon = etalon.strip('{}')
key, value = etalon.split('=')
etalon = value
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
except Exception as e:
pass
if row['CF (Is Autofilled)'] == True:
continue
prev = step_names_token[-1] if step_names_token else ""
prev_step = step_names_token[-1].split("-")[0] if step_names_token else ""
if not step_names_token or step != prev_step:
if step in options.opt_step1 or step in options.opt_step2:
new_step = step
if step in options.opt_step1[1:]:
opt1_used = True
elif step in options.opt_step2[2:]:
opt2_used = True
else:
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
step_names_token.append(new_step)
else:
if not (step in options.opt_step1 or step in options.opt_step2 or step == "FinalAnswer"):
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if prev < new_step:
step_names_token[-1] = new_step
original_steps_actions_attempts_help_levels_outcomes.append(f"{step}-{action}-{attempt}-{help_level}-{outcome}")
original_steps.append(step)
# if (not opt1_used) and (not opt2_used):
# continue
unique_steps_len = len([s for s in original_steps if not (s in options.opt_step1) and not(s in options.opt_step2)])
if step_names_token and unique_steps_len > 4:
info = ",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))])
# overall_data.append(["\t".join(step_names_token), info])
# label = None
# if opt1_used and opt2_used:
# label = "2"
# if (not opt1_used) and opt2_used:
# label = "1"
# if opt1_used and (not opt2_used):
# label = "0"
# print(f"opt1_used: {opt1_used}, opt2_used: {opt2_used} label : {label}")
# overall_labels.append(label)
proba = random.random()
# if prob in first_prob_list:
if proba <= 0.8:
train_file.write("\t".join(step_names_token))
train_file.write("\n")
# school, class, student id, progress, problem name, scenario,
# prefered ER or ME, total steps length,
# original seq-action-attempt-help_level-outcome
train_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
train_info.write("\n")
elif proba > 0.9:
# elif prob in last_prob_list:
test_file.write("\t".join(step_names_token))
test_file.write("\n")
# school, class, student id, progress, problem name, scenario,
# prefered ER or ME, total steps length,
# original seq-action-attempt-help_level-outcome
test_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
test_info.write("\n")
else:
val_file.write("\t".join(step_names_token))
val_file.write("\n")
# school, class, student id, progress, problem name, scenario,
# prefered ER or ME, total steps length,
# original seq-action-attempt-help_level-outcome
val_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
val_info.write("\n")
# break
# break
# break
# break
# break
# overall_labels = np.array(overall_labels)
# indices_of_zeros = list(np.where(overall_labels == '0')[0])
# indices_of_ones = list(np.where(overall_labels == '1')[0])
# indices_of_twos = list(np.where(overall_labels == '2')[0])
# train_len = int(len(overall_labels) * 0.10)
# sample_size = int(train_len/3)
# print(f"sample_size: {sample_size}")
# sampled_instances = random.sample(indices_of_zeros, sample_size)
# sampled_instances.extend(random.sample(indices_of_ones, sample_size))
# sampled_instances.extend(random.sample(indices_of_twos, sample_size))
# indices_of_zeros = [i for i in indices_of_zeros if not i in sampled_instances ]
# indices_of_ones = [i for i in indices_of_ones if not i in sampled_instances ]
# indices_of_twos = [i for i in indices_of_twos if not i in sampled_instances ]
# balanced_test = min(len(indices_of_zeros), len(indices_of_ones), len(indices_of_twos))
# print(f"balanced_test: {balanced_test}")
# test_sampled_instances = random.sample(indices_of_zeros, balanced_test)
# test_sampled_instances.extend(random.sample(indices_of_ones, balanced_test))
# test_sampled_instances.extend(random.sample(indices_of_twos, balanced_test))
# for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
# steps_seq = all_data[0]
# info = all_data[1]
# if index in sampled_instances:
# train_file.write(steps_seq)
# train_file.write("\n")
# train_info.write(info)
# train_info.write("\n")
# train_label.write(label)
# train_label.write("\n")
# elif index in test_sampled_instances:
# # proba = random.random()
# # if proba <0.5:
# test_file.write(steps_seq)
# test_file.write("\n")
# test_info.write(info)
# test_info.write("\n")
# test_label.write(label)
# test_label.write("\n")
# # else:
# # val_file.write(steps_seq)
# # val_file.write("\n")
# # val_info.write(info)
# # val_info.write("\n")
# # val_label.write(label)
# # val_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
val_file.close()
val_info.close()
val_label.close()
test_file.close()
test_info.close()
test_label.close()
def prepare_school_coded_finetuning_correctness_after_opts_files(data_processor, options):
'''
Ongoing research.
FinalAnswer step correctness
Correctness after opts:
0 if attempt at step>1
1 if attempt at step==1
'''
kcs = pickle.load(open("dataset/CL_2223/ratio_proportion_change3_2223/unique_kcs_list.pkl", "rb"))
kcs = [kc for kc in kcs if not pd.isna(kc)]
kcs = np.array(sorted(list(kcs)))
print(kcs, type(kcs))
print(f"KCs: {kcs}")
chunk_iterator = data_processor.load_file_iterator(sep=",")
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
# val_file = open(options.val_file_path, "w")
# val_info = open(options.val_info_path, "w")
# val_label = open(options.val_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
overall_data = []
overall_labels = []
# kcs = pickle.load(open("dataset/CL_2223/ratio_proportion_change3_2223/unique_kcs_list.pkl", "rb"))
# kcs = [kc if not pd.isna(kc) for kc in kcs]
for chunk_data in chunk_iterator:
for school, school_group in chunk_data.groupby('CF (Anon School Id)'):
if not options.school or school in options.school:
print(f"{school} : {school_group.shape}")
school_group = school_group[(school_group['CF (Is StepByStep)'] == False) &
(school_group['CF (Encounter)'] == 0) &
(school_group['CF (Is Review Mode)'] == -1) ]
print(f"{school} : {school_group.shape}")
# for class_id, class_group in school_groups.groupby('CF (Anon Class Id)'):
for student, student_groups in school_group.groupby("Anon Student Id"):
student_groups.sort_values(by="Time", inplace=True)
# prob_list = list(pd.unique(student_groups["Problem Name"]))
# prob_list= list(student_groups[student_groups["CF (Workspace Progress Status)"]=="GRADUATED"]["Problem Name"])
# prob_list = prob_list[-int(len(prob_list)/2):]
for prob, prob_groups in student_groups.groupby("Problem Name"):
# if not prob in prob_list:
# continue
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups[prob_groups["CF (Is Autofilled)"] == False]["Step Name"]))
unique_steps_len = len([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not (s in options.opt_step2)])
if unique_steps_len < 4:
continue
unique_opt_steps_len = len([s for s in unique_steps if not pd.isna(s) and (s in options.opt_step1[1:] or s in options.opt_step2[1:])])
if unique_opt_steps_len < 2:
continue
# print(unique_steps, unique_opt_steps_len)
class_id = list(pd.unique(prob_groups["CF (Anon Class Id)"]))
step_names_token = []
original_steps_actions_attempts_help_levels_outcomes = []
original_steps = []
means_and_extremes = False
opt1_used = False
opt2_used = False
final_after_opts = False
correctness = "0"
kcs_skills = [0 for i in kcs]
diff_skills = [0 for i in kcs]
finalanswer_skill = [0 for i in kcs]
for index, row in prob_groups[['Step Name', 'Action', 'Attempt At Step', 'CF (Is Autofilled)',
'Outcome', 'Help Level', 'CF (Workspace Progress Status)',
'CF (Skill Previous p-Known)', 'CF (Skill New p-Known)', 'KC Model(MATHia)',
'CF (Etalon)', 'CF (Problem Scenario Tags)']].iterrows():
step = row["Step Name"]
action = row["Action"] # ['Attempt', 'Hint Request', 'Hint Level Change', 'Done']
attempt = row["Attempt At Step"] # number
outcome = row["Outcome"] # ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
help_level = row["Help Level"] # number
progress = row["CF (Workspace Progress Status)"]
scenario = row['CF (Problem Scenario Tags)']
kc = row['KC Model(MATHia)']
prev_skill = row['CF (Skill Previous p-Known)']
curr_skill = row['CF (Skill New p-Known)']
# print(kc, prev_skill)
if not pd.isna(step):
if step in options.opt_step1 and not means_and_extremes:
etalon = row["CF (Etalon)"]
if not pd.isna(etalon):
etalon = etalon.strip('{}')
key, value = etalon.split('=')
etalon = value
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
except Exception as e:
pass
if row['CF (Is Autofilled)'] == True:
continue
prev = step_names_token[-1] if step_names_token else ""
prev_step = step_names_token[-1].split("-")[0] if step_names_token else ""
if not step_names_token or step != prev_step:
if step in options.opt_step1 or step in options.opt_step2:
new_step = step
if step in options.opt_step1[1:]:
opt1_used = True
elif step in options.opt_step2[2:]:
opt2_used = True
else:
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if step == "FinalAnswer" and (opt1_used or opt2_used) and not final_after_opts:
final_after_opts = True
if outcome == "OK":
correctness = "1"
step_names_token.append(new_step)
else:
if not (step in options.opt_step1 or step in options.opt_step2 or step == "FinalAnswer"):
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if prev < new_step:
step_names_token[-1] = new_step
if not pd.isna(kc):
index = np.argwhere(kcs==kc).flatten()[0]
# print(index, type(index))
kcs_skills[index] = prev_skill
diff_skills[index] = prev_skill - curr_skill
if step == "FinalAnswer":
finalanswer_skill[index] = prev_skill
original_steps_actions_attempts_help_levels_outcomes.append(f"{step}-{action}-{attempt}-{help_level}-{outcome}")
original_steps.append(step)
if (not opt1_used) and (not opt2_used):
continue
unique_steps_len = len([s for s in original_steps if not (s in options.opt_step1) and not(s in options.opt_step2)])
if step_names_token and unique_steps_len > 4:
label = None
if opt1_used and opt2_used:
label = "2"
if (not opt1_used) and opt2_used:
label = "1"
if opt1_used and (not opt2_used):
label = "0"
# print(f"opt1_used: {opt1_ßused}, opt2_used: {opt2_used} label : {label}")
info = ",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes)), label,
"\t".join(map(str, kcs_skills)), "\t".join(map(str, diff_skills)),
"\t".join(map(str, finalanswer_skill))])#str(finalanswer_skill)])
overall_data.append(["\t".join(step_names_token), info])
overall_labels.append(correctness)
# proba = random.random()
# # if prob in first_prob_list:
# if proba <= 0.8:
# train_file.write("\t".join(step_names_token))
# train_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# train_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# train_info.write("\n")
# elif proba > 0.9:
# # elif prob in last_prob_list:
# test_file.write("\t".join(step_names_token))
# test_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# test_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# test_info.write("\n")
# else:
# val_file.write("\t".join(step_names_token))
# val_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# val_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# val_info.write("\n")
# break
# break
# break
# break
# break
overall_labels = np.array(overall_labels)
indices_of_zeros = list(np.where(overall_labels == '0')[0])
indices_of_ones = list(np.where(overall_labels == '1')[0])
# indices_of_twos = list(np.where(overall_labels == '2')[0])
train_len = int(len(overall_labels) * 0.10)
sample_size = int(train_len/2)
print(f"sample_size: {sample_size}")
sampled_instances = random.sample(indices_of_zeros, sample_size)
sampled_instances.extend(random.sample(indices_of_ones, sample_size))
# sampled_instances.extend(random.sample(indices_of_twos, sample_size))
indices_of_zeros = [i for i in indices_of_zeros if not i in sampled_instances ]
indices_of_ones = [i for i in indices_of_ones if not i in sampled_instances ]
# indices_of_twos = [i for i in indices_of_twos if not i in sampled_instances ]
balanced_test = min(len(indices_of_zeros), len(indices_of_ones)) #, len(indices_of_twos))
print(f"balanced_test: {balanced_test}")
test_sampled_instances = random.sample(indices_of_zeros, balanced_test)
test_sampled_instances.extend(random.sample(indices_of_ones, balanced_test))
# test_sampled_instances.extend(random.sample(indices_of_twos, balanced_test))
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
steps_seq = all_data[0]
info = all_data[1]
if index in sampled_instances:
train_file.write(steps_seq)
train_file.write("\n")
train_info.write(info)
train_info.write("\n")
train_label.write(label)
train_label.write("\n")
elif index in test_sampled_instances:
# proba = random.random()
# if proba <0.5:
test_file.write(steps_seq)
test_file.write("\n")
test_info.write(info)
test_info.write("\n")
test_label.write(label)
test_label.write("\n")
# else:
# val_file.write(steps_seq)
# val_file.write("\n")
# val_info.write(info)
# val_info.write("\n")
# val_label.write(label)
# val_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
# val_file.close()
# val_info.close()
# val_label.close()
test_file.close()
test_info.close()
test_label.close()
def prepare_school_coded_finetuning_correctness_after_opts_over_prob_files(data_processor, options):
'''
Ongoing research.
FinalAnswer step correctness
Correctness after opts:
0 if attempt at step>1
1 if attempt at step==1
'''
kcs = pickle.load(open("dataset/CL_2223/ratio_proportion_change3_2223/unique_kcs_list.pkl", "rb"))
kcs = [kc for kc in kcs if not pd.isna(kc)]
kcs = np.array(sorted(list(kcs)))
print(kcs, type(kcs))
print(f"KCs: {kcs}")
chunk_iterator = data_processor.load_file_iterator(sep=",")
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
# val_file = open(options.val_file_path, "w")
# val_info = open(options.val_info_path, "w")
# val_label = open(options.val_label_path, "w")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
train_data = []
train_labels = []
test_data = []
test_labels = []
# kcs = pickle.load(open("dataset/CL_2223/ratio_proportion_change3_2223/unique_kcs_list.pkl", "rb"))
# kcs = [kc if not pd.isna(kc) for kc in kcs]
for chunk_data in chunk_iterator:
for school, school_group in chunk_data.groupby('CF (Anon School Id)'):
if not options.school or school in options.school:
print(f"{school} : {school_group.shape}")
school_group = school_group[(school_group['CF (Is StepByStep)'] == False) &
(school_group['CF (Encounter)'] == 0) &
(school_group['CF (Is Review Mode)'] == -1) ]
print(f"{school} : {school_group.shape}")
# for class_id, class_group in school_groups.groupby('CF (Anon Class Id)'):
for student, student_groups in school_group.groupby("Anon Student Id"):
student_groups.sort_values(by="Time", inplace=True)
train = True
proba = random.random()
if proba < 0.5:
train = False
# prob_list = list(pd.unique(student_groups["Problem Name"]))
# prob_list= list(student_groups[student_groups["CF (Workspace Progress Status)"]=="GRADUATED"]["Problem Name"])
# prob_list = prob_list[-int(len(prob_list)/2):]
prev_kcs_skills = [0 for i in kcs]
for pi, (prob, prob_groups) in enumerate(student_groups.groupby("Problem Name")):
# if not prob in prob_list:
# continue
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups[prob_groups["CF (Is Autofilled)"] == False]["Step Name"]))
unique_steps_len = len([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not (s in options.opt_step2)])
if unique_steps_len < 4:
continue
unique_opt_steps_len = len([s for s in unique_steps if not pd.isna(s) and (s in options.opt_step1[1:] or s in options.opt_step2[1:])])
if unique_opt_steps_len < 2:
continue
# print(unique_steps, unique_opt_steps_len)
class_id = list(pd.unique(prob_groups["CF (Anon Class Id)"]))
step_names_token = []
original_steps_actions_attempts_help_levels_outcomes = []
original_steps = []
means_and_extremes = False
opt1_used = False
opt2_used = False
final_after_opts = False
correctness = "0"
kcs_skills = [0 for i in kcs]
diff_skills = [0 for i in kcs]
finalanswer_skill = [0 for i in kcs]
for index, row in prob_groups[['Step Name', 'Action', 'Attempt At Step', 'CF (Is Autofilled)',
'Outcome', 'Help Level', 'CF (Workspace Progress Status)',
'CF (Skill Previous p-Known)', 'CF (Skill New p-Known)', 'KC Model(MATHia)',
'CF (Etalon)', 'CF (Problem Scenario Tags)']].iterrows():
step = row["Step Name"]
action = row["Action"] # ['Attempt', 'Hint Request', 'Hint Level Change', 'Done']
attempt = row["Attempt At Step"] # number
outcome = row["Outcome"] # ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
help_level = row["Help Level"] # number
progress = row["CF (Workspace Progress Status)"]
scenario = row['CF (Problem Scenario Tags)']
kc = row['KC Model(MATHia)']
prev_skill = row['CF (Skill Previous p-Known)']
curr_skill = row['CF (Skill New p-Known)']
# print(kc, prev_skill)
if not pd.isna(step):
if step in options.opt_step1 and not means_and_extremes:
etalon = row["CF (Etalon)"]
if not pd.isna(etalon):
etalon = etalon.strip('{}')
key, value = etalon.split('=')
etalon = value
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
except Exception as e:
pass
if row['CF (Is Autofilled)'] == True:
continue
prev = step_names_token[-1] if step_names_token else ""
prev_step = step_names_token[-1].split("-")[0] if step_names_token else ""
if not step_names_token or step != prev_step:
if step in options.opt_step1 or step in options.opt_step2:
new_step = step
if step in options.opt_step1[1:]:
opt1_used = True
elif step in options.opt_step2[2:]:
opt2_used = True
else:
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if step == "FinalAnswer" and (opt1_used or opt2_used) and not final_after_opts:
final_after_opts = True
if outcome == "OK":
correctness = "1"
step_names_token.append(new_step)
else:
if not (step in options.opt_step1 or step in options.opt_step2 or step == "FinalAnswer"):
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if prev < new_step:
step_names_token[-1] = new_step
if not pd.isna(kc):
index = np.argwhere(kcs==kc).flatten()[0]
# print(index, type(index))
kcs_skills[index] = prev_skill
if pi != 0:
diff_skills[index] = prev_skill - prev_kcs_skills[index]
prev_kcs_skills[index] = prev_skill
if step == "FinalAnswer":
finalanswer_skill[index] = prev_skill
original_steps_actions_attempts_help_levels_outcomes.append(f"{step}-{action}-{attempt}-{help_level}-{outcome}")
original_steps.append(step)
if (not opt1_used) and (not opt2_used):
continue
unique_steps_len = len([s for s in original_steps if not (s in options.opt_step1) and not(s in options.opt_step2)])
if step_names_token and unique_steps_len > 4:
label = None
if opt1_used and opt2_used:
label = "2"
if (not opt1_used) and opt2_used:
label = "1"
if opt1_used and (not opt2_used):
label = "0"
# print(f"opt1_used: {opt1_ßused}, opt2_used: {opt2_used} label : {label}")
info = ",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes)), label,
"\t".join(map(str, kcs_skills)), "\t".join(map(str, diff_skills)),
"\t".join(map(str, finalanswer_skill))])#str(finalanswer_skill)])
if train:
train_data.append(["\t".join(step_names_token), info])
train_labels.append(correctness)
else:
test_data.append(["\t".join(step_names_token), info])
test_labels.append(correctness)
# proba = random.random()
# # if prob in first_prob_list:
# if proba <= 0.8:
# train_file.write("\t".join(step_names_token))
# train_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# train_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# train_info.write("\n")
# elif proba > 0.9:
# # elif prob in last_prob_list:
# test_file.write("\t".join(step_names_token))
# test_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# test_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# test_info.write("\n")
# else:
# val_file.write("\t".join(step_names_token))
# val_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# val_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# val_info.write("\n")
# break
# break
# break
# break
# break
# overall_labels = np.array(overall_labels)
# indices_of_zeros = list(np.where(overall_labels == '0')[0])
# indices_of_ones = list(np.where(overall_labels == '1')[0])
# # indices_of_twos = list(np.where(overall_labels == '2')[0])
# train_len = int(len(overall_labels) * 0.10)
# sample_size = int(train_len/2)
# print(f"sample_size: {sample_size}")
# sampled_instances = random.sample(indices_of_zeros, sample_size)
# sampled_instances.extend(random.sample(indices_of_ones, sample_size))
# # sampled_instances.extend(random.sample(indices_of_twos, sample_size))
# indices_of_zeros = [i for i in indices_of_zeros if not i in sampled_instances ]
# indices_of_ones = [i for i in indices_of_ones if not i in sampled_instances ]
# # indices_of_twos = [i for i in indices_of_twos if not i in sampled_instances ]
# balanced_test = min(len(indices_of_zeros), len(indices_of_ones)) #, len(indices_of_twos))
# print(f"balanced_test: {balanced_test}")
# test_sampled_instances = random.sample(indices_of_zeros, balanced_test)
# test_sampled_instances.extend(random.sample(indices_of_ones, balanced_test))
# # test_sampled_instances.extend(random.sample(indices_of_twos, balanced_test))
for index, (all_data, label) in enumerate(zip(train_data, train_labels)):
steps_seq = all_data[0]
info = all_data[1]
train_file.write(steps_seq)
train_file.write("\n")
train_info.write(info)
train_info.write("\n")
train_label.write(label)
train_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
for index, (all_data, label) in enumerate(zip(test_data, test_labels)):
steps_seq = all_data[0]
info = all_data[1]
test_file.write(steps_seq)
test_file.write("\n")
test_info.write(info)
test_info.write("\n")
test_label.write(label)
test_label.write("\n")
test_file.close()
test_info.close()
test_label.close()
def prepare_school_coded_finetuning_correctness_after_opts_per_files(data_processor, options):
'''
Ongoing research.
FinalAnswer step correctness
Correctness after opts:
0 if attempt at step>1
1 if attempt at step==1
'''
kcs = pickle.load(open("dataset/CL_2223/ratio_proportion_change3_2223/unique_kcs_list.pkl", "rb"))
kcs = [kc for kc in kcs if not pd.isna(kc)]
kcs = np.array(sorted(list(kcs)))
print(kcs, type(kcs))
print(f"KCs: {kcs}")
chunk_iterator = data_processor.load_file_iterator(sep=",")
train_file = open(options.train_file_path, "w")
train_info = open(options.train_info_path, "w")
train_label = open(options.train_label_path, "w")
val_file = open(options.val_file_path, "a")
val_info = open(options.val_info_path, "a")
val_label = open(options.val_label_path, "a")
test_file = open(options.test_file_path, "w")
test_info = open(options.test_info_path, "w")
test_label = open(options.test_label_path, "w")
overall_data = []
overall_labels = []
# kcs = pickle.load(open("dataset/CL_2223/ratio_proportion_change3_2223/unique_kcs_list.pkl", "rb"))
# kcs = [kc if not pd.isna(kc) for kc in kcs]
for chunk_data in chunk_iterator:
for school, school_group in chunk_data.groupby('CF (Anon School Id)'):
if not options.school or school in options.school:
print(f"{school} : {school_group.shape}")
school_group = school_group[(school_group['CF (Is StepByStep)'] == False) &
(school_group['CF (Encounter)'] == 0) &
(school_group['CF (Is Review Mode)'] == -1) ]
print(f"{school} : {school_group.shape}")
# for class_id, class_group in school_groups.groupby('CF (Anon Class Id)'):
for student, student_groups in school_group.groupby("Anon Student Id"):
student_groups.sort_values(by="Time", inplace=True)
# prob_list = list(pd.unique(student_groups["Problem Name"]))
# prob_list= list(student_groups[student_groups["CF (Workspace Progress Status)"]=="GRADUATED"]["Problem Name"])
# prob_list = prob_list[-int(len(prob_list)/2):]
for prob, prob_groups in student_groups.groupby("Problem Name"):
# if not prob in prob_list:
# continue
actions = list(prob_groups["Action"])
# A problem should be completed by a student clicking Done button.
if not "Done" in actions:
continue
unique_steps = list(pd.unique(prob_groups[prob_groups["CF (Is Autofilled)"] == False]["Step Name"]))
unique_steps_len = len([s for s in unique_steps if not pd.isna(s) and not (s in options.opt_step1) and not (s in options.opt_step2)])
if unique_steps_len < 4:
continue
unique_opt_steps_len = len([s for s in unique_steps if not pd.isna(s) and (s in options.opt_step1[1:] or s in options.opt_step2[1:])])
if unique_opt_steps_len < 2:
continue
# print(unique_steps, unique_opt_steps_len)
class_id = list(pd.unique(prob_groups["CF (Anon Class Id)"]))
step_names_token = []
original_steps_actions_attempts_help_levels_outcomes = []
original_steps = []
means_and_extremes = False
opt1_used = False
opt2_used = False
final_after_opts = False
correctness = "0"
kcs_skills = [0 for i in kcs]
diff_skills = [0 for i in kcs]
finalanswer_skill = [0 for i in kcs]
for index, row in prob_groups[['Step Name', 'Action', 'Attempt At Step', 'CF (Is Autofilled)',
'Outcome', 'Help Level', 'CF (Workspace Progress Status)',
'CF (Skill Previous p-Known)', 'CF (Skill New p-Known)', 'KC Model(MATHia)',
'CF (Etalon)', 'CF (Problem Scenario Tags)']].iterrows():
step = row["Step Name"]
action = row["Action"] # ['Attempt', 'Hint Request', 'Hint Level Change', 'Done']
attempt = row["Attempt At Step"] # number
outcome = row["Outcome"] # ['OK', 'BUG', 'ERROR', 'INITIAL_HINT', 'HINT_LEVEL_CHANGE']
help_level = row["Help Level"] # number
progress = row["CF (Workspace Progress Status)"]
scenario = row['CF (Problem Scenario Tags)']
kc = row['KC Model(MATHia)']
prev_skill = row['CF (Skill Previous p-Known)']
curr_skill = row['CF (Skill New p-Known)']
# print(kc, prev_skill)
if not pd.isna(step):
if step in options.opt_step1 and not means_and_extremes:
etalon = row["CF (Etalon)"]
if not pd.isna(etalon):
etalon = etalon.strip('{}')
key, value = etalon.split('=')
etalon = value
try:
etalon = int(etalon)
except Exception as e:
try:
etalon = float(etalon)
means_and_extremes = True
except Exception as e:
pass
if row['CF (Is Autofilled)'] == True:
continue
prev = step_names_token[-1] if step_names_token else ""
prev_step = step_names_token[-1].split("-")[0] if step_names_token else ""
if not step_names_token or step != prev_step:
if step in options.opt_step1 or step in options.opt_step2:
new_step = step
if step in options.opt_step1[1:]:
opt1_used = True
elif step in options.opt_step2[2:]:
opt2_used = True
else:
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if step == "FinalAnswer" and (opt1_used or opt2_used) and not final_after_opts:
final_after_opts = True
if outcome == "OK":
correctness = "1"
step_names_token.append(new_step)
else:
if not (step in options.opt_step1 or step in options.opt_step2 or step == "FinalAnswer"):
if action == "Attempt" and outcome != "OK":
new_step = step+"-2"
elif "Hint" in action:
new_step = step+"-1"
else:
new_step = step+"-0"
if prev < new_step:
step_names_token[-1] = new_step
if not pd.isna(kc):
index = np.argwhere(kcs==kc).flatten()[0]
# print(index, type(index))
kcs_skills[index] = prev_skill
diff_skills[index] = prev_skill - curr_skill
if step == "FinalAnswer":
finalanswer_skill[index] = prev_skill
original_steps_actions_attempts_help_levels_outcomes.append(f"{step}-{action}-{attempt}-{help_level}-{outcome}")
original_steps.append(step)
if (not opt1_used) and (not opt2_used):
continue
unique_steps_len = len([s for s in original_steps if not (s in options.opt_step1) and not(s in options.opt_step2)])
if step_names_token and unique_steps_len > 4:
label = None
if opt1_used and opt2_used:
label = "2"
if (not opt1_used) and opt2_used:
label = "1"
if opt1_used and (not opt2_used):
label = "0"
# print(f"opt1_used: {opt1_ßused}, opt2_used: {opt2_used} label : {label}")
info = ",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
"\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes)), label,
"\t".join(map(str, kcs_skills)), "\t".join(map(str, diff_skills)),
"\t".join(map(str, finalanswer_skill))])#str(finalanswer_skill)])
overall_data.append(["\t".join(step_names_token), info])
overall_labels.append(correctness)
# proba = random.random()
# # if prob in first_prob_list:
# if proba <= 0.8:
# train_file.write("\t".join(step_names_token))
# train_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# train_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# train_info.write("\n")
# elif proba > 0.9:
# # elif prob in last_prob_list:
# test_file.write("\t".join(step_names_token))
# test_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# test_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# test_info.write("\n")
# else:
# val_file.write("\t".join(step_names_token))
# val_file.write("\n")
# # school, class, student id, progress, problem name, scenario,
# # prefered ER or ME, total steps length,
# # original seq-action-attempt-help_level-outcome
# val_info.write(",".join([str(school), "\t".join(class_id), str(student), str(progress), str(prob), str(scenario),
# f"{1 if means_and_extremes else 0}", str(len(step_names_token)),
# "\t".join(map(str, original_steps_actions_attempts_help_levels_outcomes))]))
# val_info.write("\n")
# break
# break
# break
# break
# break
overall_labels = np.array(overall_labels)
indices_of_zeros = list(np.where(overall_labels == '0')[0])
indices_of_ones = list(np.where(overall_labels == '1')[0])
# indices_of_twos = list(np.where(overall_labels == '2')[0])
# train_len = int(len(overall_labels) * 0.10)
train_len = int(len(overall_labels) * float(options.per))
sample_size = int(train_len/2)
if float(options.per) == 1:
sample_size = min(len(indices_of_zeros), len(indices_of_ones))
elif float(options.per) > 1:
sample_size = int(options.per)
print(f"sample_size: {sample_size}")
sampled_instances = random.sample(indices_of_zeros, sample_size)
sampled_instances.extend(random.sample(indices_of_ones, sample_size))
# sampled_instances.extend(random.sample(indices_of_twos, sample_size))
indices_of_zeros = [i for i in indices_of_zeros if not i in sampled_instances ]
indices_of_ones = [i for i in indices_of_ones if not i in sampled_instances ]
# indices_of_twos = [i for i in indices_of_twos if not i in sampled_instances ]
balanced_test = min(len(indices_of_zeros), len(indices_of_ones)) #, len(indices_of_twos))
print(f"balanced_test: {balanced_test}")
test_sampled_instances = random.sample(indices_of_zeros, balanced_test)
test_sampled_instances.extend(random.sample(indices_of_ones, balanced_test))
# test_sampled_instances.extend(random.sample(indices_of_twos, balanced_test))
for index, (all_data, label) in enumerate(zip(overall_data, overall_labels)):
steps_seq = all_data[0]
info = all_data[1]
if index in sampled_instances:
train_file.write(steps_seq)
train_file.write("\n")
train_info.write(info)
train_info.write("\n")
train_label.write(label)
train_label.write("\n")
if float(options.per) == 1.0:
val_file.write(steps_seq)
val_file.write("\n")
val_info.write(info)
val_info.write("\n")
val_label.write(label)
val_label.write("\n")
elif index in test_sampled_instances:
# proba = random.random()
# if proba <0.5:
test_file.write(steps_seq)
test_file.write("\n")
test_info.write(info)
test_info.write("\n")
test_label.write(label)
test_label.write("\n")
if float(options.per) != 1.0:
val_file.write(steps_seq)
val_file.write("\n")
val_info.write(info)
val_info.write("\n")
val_label.write(label)
val_label.write("\n")
train_file.close()
train_info.close()
train_label.close()
val_file.close()
val_info.close()
val_label.close()
test_file.close()
test_info.close()
test_label.close()
def prepare_pretraining_vocab_file(options):
# kc = pickle.load(open("dataset/unique/unique_kcs_list.pkl","rb"))
# kc_token = {"KC"+str(i):k for i, k in enumerate(kc)}
# pickle.dump(kc_token, open("pretraining/unique_dict_kc_token.pkl", "wb"))
# steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
# step_token = {"step"+str(i):k for i, k in enumerate(steps)}
# folder_name = options.workspace_name+"/" if options.workspace_name else ""
# pickle.dump(step_token, open(f"{folder_name}pretraining/unique_dict_step_token.pkl", "wb"))
# steps = pickle.load(open(f"{options.dataset_folder}unique_new_steps_w_action_attempt_list.pkl","rb"))
steps = pickle.load(open(f"{options.dataset_folder}unique_steps_list.pkl","rb"))
# print("No of unique kc", len(kc))
print("No of unique steps ", len(steps))
# print("No of unique problem", len(prob))
# print("Size of vocab ", len(steps))
ordered_steps = sorted(list(steps))
with (open(options.vocab_file_path,"w")) as vb_file:
vb_file.write("[PAD]\n")
vb_file.write("[UNK]\n")
vb_file.write("[MASK]\n")
vb_file.write("[CLS]\n")
vb_file.write("[SEP]\n")
# vb_file.write("\n".join(kc_token.keys()))
# vb_file.write("\n")
# vb_file.write("\n".join(step_token.keys()))
# vb_file.write("\n".join(ordered_steps))
for step in ordered_steps:
if step in options.opt_step1 or step in options.opt_step2:
vb_file.write(f"{step}\n")
else:
for i in range(3):
vb_file.write(f"{step}-{i}\n")
vb_file.close()
with open(options.vocab_file_path,"r") as f:
l = f.readlines()
print(l, len(l))
f.close()
def main(opt):
options = copy.deepcopy(opt)
if opt.workspace_name:
options.dataset_folder = opt.dataset_folder+opt.workspace_name+"/"
data_processor = DataPreprocessor(input_file_path=opt.dataset)
if opt.analyze_dataset_by_section:
print(f"Analyzing dataset by section for workspace: {opt.workspace_name}")
data_processor.analyze_dataset_by_section(opt.workspace_name)
pickle.dump(data_processor.unique_students, open(f"{options.dataset_folder}unique_students_list.pkl", "wb"))
pickle.dump(data_processor.unique_problems, open(f"{options.dataset_folder}unique_problems_list.pkl", "wb"))
pickle.dump(data_processor.unique_prob_hierarchy, open(f"{options.dataset_folder}unique_hierarchy_list.pkl", "wb"))
pickle.dump(data_processor.unique_kcs, open(f"{options.dataset_folder}unique_kcs_list.pkl", "wb"))
pickle.dump(data_processor.unique_steps, open(f"{options.dataset_folder}unique_steps_list.pkl", "wb"))
if opt.analyze_dataset_by_school:
print(f"Analyzing dataset of all school for workspace: {opt.workspace_name}")
data_processor.analyze_dataset_by_school(opt.workspace_name)
if not os.path.exists(options.dataset_folder):
os.makedirs(options.dataset_folder)
pickle.dump(data_processor.unique_schools, open(f"{options.dataset_folder}unique_schools_list.pkl", "wb"))
pickle.dump(data_processor.unique_class, open(f"{options.dataset_folder}unique_class_list.pkl", "wb"))
pickle.dump(data_processor.unique_students, open(f"{options.dataset_folder}unique_students_list.pkl", "wb"))
pickle.dump(data_processor.unique_problems, open(f"{options.dataset_folder}unique_problems_list.pkl", "wb"))
pickle.dump(data_processor.unique_kcs, open(f"{options.dataset_folder}unique_kcs_list.pkl", "wb"))
pickle.dump(data_processor.unique_steps, open(f"{options.dataset_folder}unique_steps_list.pkl", "wb"))
pickle.dump(data_processor.unique_new_steps_w_action_attempt, open(f"{options.dataset_folder}unique_new_steps_w_action_attempt_list.pkl", "wb"))
pickle.dump(data_processor.unique_new_steps_w_action_attempt_kcs, open(f"{options.dataset_folder}unique_new_steps_w_action_attempt_kcs.pkl", "wb"))
pickle.dump(data_processor.unique_new_steps_w_kcs, open(f"{options.dataset_folder}unique_new_steps_w_kcs_list.pkl", "wb"))
if opt.workspace_name:
for k,v in vars(opt).items():
if 'path' in k:
if v:
redirect_path = opt.workspace_name+"/"
if opt.school and opt.pretrain:
sch = f"sch_largest_{len(opt.school)}-coded" #f"sch_largest_655"
redirect_path = redirect_path + sch+"/"
if opt.school_folder:
redirect_path = redirect_path + opt.school_folder+"/"
# else:
# sch = "sch_largest_655"
if k != "vocab_file_path":
if opt.pretrain:
redirect_path = redirect_path + "pretraining/"
else:
if opt.code:
redirect_path = redirect_path + f"{opt.code}/"
elif opt.finetune_task:
if opt.diff_val_folder and "val" in v:
redirect_path = redirect_path + f"finetuning/"
else:
redirect_path = redirect_path + f"finetuning/{opt.finetune_task}/"
if not os.path.exists(redirect_path):
os.makedirs(redirect_path)
else:
if not os.path.exists(redirect_path+"/pretraining/"):
os.makedirs(redirect_path+"/pretraining/")
setattr(options, f"{k}", redirect_path+v)
# setattr(options, f"{k}", opt.workspace_name+"/check/"+v)
print(f"options.{k} : {getattr(options, f'{k}')}")
if options.pretrain:
print("Preparing vocab...")
prepare_pretraining_vocab_file(options)
print("Preparing pre-training dataset...")
# old non-repeated steps
# prepare_pretraining_files(data_processor, options)
# coded
# prepare_school_coded_pretraining_files(data_processor, options)
prepare_school_coded_finetuning_opts_intentional_files(data_processor, options)
# prepare_pretraining_files(data_processor, options)
# prepare_school_pretraining_files(data_processor, options)
# else:
# print("Preparing attention dataset...")
# prepare_school_attention_files(data_processor, options)
else:
print("Preparing fine-tuning dataset...")
# _1920
# prepare_finetuning_10per_files(data_processor, options)
# prepare_finetuning_IS_FS_files(data_processor, options)
# prepare_finetuning_correctness_files(data_processor, options)
# _2223
# prepare_school_coded_finetuning_partial_seq_files(data_processor, options)
# prepare_school_coded_finetuning_opts_files(data_processor, options)
prepare_school_coded_finetuning_correctness_after_opts_per_files(data_processor, options)
# prepare_school_coded_finetuning_correctness_after_opts_files(data_processor, options)
# prepare_school_coded_finetuning_correctness_after_opts_over_prob_files(data_processor, options)
# prepare_finetuning_IS_files(data_processor, options)
# # prepare_finetuning_FS_files(data_processor, options)
# prepare_finetuning_correctness_aaai_files(data_processor, options)
# # prepare_finetuning_SL_files(data_processor, options)
# # prepare_finetuning_effectiveness_files(data_processor, options)
# prepare_attn_test_files(data_processor, options)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-dataset_folder', type=str, default="dataset/CL4999_1920/")
parser.add_argument('-analyze_dataset_by_section', type=bool, default=False)
parser.add_argument('-analyze_dataset_by_school', type=bool, default=False)
parser.add_argument('-workspace_name', type=str, default=None)
parser.add_argument('-school', nargs='+', type=str, default=None)
parser.add_argument('-school_folder', type=str, default=None)
# parser.add_argument('-highGRschool', nargs='+', type=str, default=None)
# parser.add_argument('-lowGRschool', nargs='+', type=str, default=None)
parser.add_argument('-code', type=str, default=None)
parser.add_argument('-finetune_task', type=str, default=None)
parser.add_argument('-per', type=float, default=None)
parser.add_argument("-diff_val_folder", type=bool, default=False, help="use for different val folder")
parser.add_argument('-opt_step1', nargs='+', type=str, help='List of optional steps 1')
parser.add_argument('-opt_step2', nargs='+', type=str, help='List of optional steps 2')
parser.add_argument('-final_step', nargs='+', type=str, help='List of final step')
parser.add_argument('-dataset', type=str, default="dataset/CL4999_1920/course2_1920_4999_students_datashop.txt")
parser.add_argument('-pretrain', type=bool, default=False)
parser.add_argument('-vocab_file_path', type=str, default="pretraining/vocab.txt") #pretraining/vocab.txt
# Prepare for pretraining
parser.add_argument('-train_file_path', type=str, default="train.txt") #pretraining/pretrain.txt
parser.add_argument('-train_info_path', type=str, default="train_info.txt") #pretraining/pretrain_info.txt
parser.add_argument('-train_label_path', type=str, default="train_label.txt") #finetuning/train_label.txt
parser.add_argument('-val_file_path', type=str, default="val.txt") #pretraining/val.txt
parser.add_argument('-val_info_path', type=str, default="val_info.txt") #pretraining/val_info.txt
parser.add_argument('-val_label_path', type=str, default="val_label.txt") #finetuning/val_label.txt
parser.add_argument('-test_file_path', type=str, default="test.txt") #pretraining/test.txt
parser.add_argument('-test_info_path', type=str, default="test_info.txt") #pretraining/test_info.txt
parser.add_argument('-test_label_path', type=str, default="test_label.txt") #finetuning/test_label.txt
# parser.add_argument('-train_gt_label_path', type=str, default="finetuning/train_gt_label.txt")
# parser.add_argument('-test_gt_label_path', type=str, default="finetuning/test_gt_label.txt")
options = parser.parse_args()
if not options.opt_step1:
setattr(options, "opt_step1", [])
print("Optional steps 1: ", options.opt_step1)
if not options.opt_step2:
setattr(options, "opt_step2", [])
print("Optional steps 2: ", options.opt_step2)
if not options.final_step:
setattr(options, "final_step", [])
print("Final steps: ", options.final_step)
main(options)