def parse_lm(input_file, prefix_len=None, bpe_symbol=None, target_prefix_frac=None):
    """parse output of eval_lm"""
    with open(input_file, "r") as f:
        text = f.readlines()
        text = text[7:]
        cleaned_text = text[:-2]

        sentences = {}
        sen_scores = {}
        sen_pos_scores = {}
        no_bpe_sentences = {}
        num_bpe_tokens_dict = {}
        for _i, line in enumerate(cleaned_text):
            tokens = line.split()
            if tokens[0].isdigit():
                line_id = int(tokens[0])
                scores = [float(x[1:-1]) for x in tokens[2::2]]
                sentences[line_id] = " ".join(tokens[1::2][:-1]) + "\n"
                if bpe_symbol is not None:
                    # exclude <eos> symbol to match output from generate.py
                    bpe_sen = " ".join(tokens[1::2][:-1]) + "\n"
                    no_bpe_sen = remove_bpe(bpe_sen, bpe_symbol)
                    no_bpe_sentences[line_id] = no_bpe_sen

                if prefix_len is not None:
                    num_bpe_tokens = get_num_bpe_tokens_from_len(
                        bpe_sen, bpe_symbol, prefix_len
                    )
                    sen_scores[line_id] = sum(scores[:num_bpe_tokens])
                    num_bpe_tokens_dict[line_id] = num_bpe_tokens
                elif target_prefix_frac is not None:
                    num_words, shortened, target_prefix_len = calc_length_from_frac(
                        bpe_sen, target_prefix_frac, bpe_symbol
                    )
                    sen_scores[line_id] = sum(scores[:target_prefix_len])
                    num_bpe_tokens_dict[line_id] = target_prefix_len
                else:
                    sen_scores[line_id] = sum(scores)
                    num_bpe_tokens_dict[line_id] = len(scores)

                sen_pos_scores[line_id] = scores

    return sentences, sen_scores, sen_pos_scores, no_bpe_sentences, num_bpe_tokens_dict
parse output of eval_lm
examples/noisychannel/rerank_utils.py