def get_examples(data_dir, set_type):
    """
    Extract paragraph and question-answer list from each json file
    """
    examples = []

    levels = ["middle", "high"]
    set_type_c = set_type.split("-")
    if len(set_type_c) == 2:
        levels = [set_type_c[1]]
        set_type = set_type_c[0]
    for level in levels:
        cur_dir = os.path.join(data_dir, set_type, level)
        for filename in os.listdir(cur_dir):
            cur_path = os.path.join(cur_dir, filename)
            with open(cur_path, "r") as f:
                cur_data = json.load(f)
                answers = cur_data["answers"]
                options = cur_data["options"]
                questions = cur_data["questions"]
                context = cur_data["article"].replace("\n", " ")
                context = re.sub(r"\s+", " ", context)
                for i in range(len(answers)):
                    label = ord(answers[i]) - ord("A")
                    qa_list = []
                    question = questions[i]
                    for j in range(4):
                        option = options[i][j]
                        if "_" in question:
                            qa_cat = question.replace("_", option)
                        else:
                            qa_cat = " ".join([question, option])
                        qa_cat = re.sub(r"\s+", " ", qa_cat)
                        qa_list.append(qa_cat)
                    examples.append(InputExample(context, qa_list, label))

    return examples
Extract paragraph and question-answer list from each json file
examples/roberta/preprocess_RACE.py