    def __init__(
        self,
        output_file,
        backwards,
        right_to_left,
        bpe_symbol,
        prefix_len=None,
        target_prefix_frac=None,
        source_prefix_frac=None,
    ):
        """process output from rescoring"""
        source, hypo, score, target, pos_score = reprocess(output_file)
        if backwards:
            self.hypo_fracs = source_prefix_frac
        else:
            self.hypo_fracs = target_prefix_frac

        # remove length penalty so we can use raw scores
        score, num_bpe_tokens = get_score_from_pos(
            pos_score, prefix_len, hypo, bpe_symbol, self.hypo_fracs, backwards
        )
        source_lengths = {}
        target_lengths = {}

        assert hypo.keys() == source.keys(), "key mismatch"
        if backwards:
            tmp = hypo
            hypo = source
            source = tmp
        for i in source:
            # since we are reranking, there should only be one hypo per source sentence
            if backwards:
                len_src = len(source[i][0].split())
                # record length without <eos>
                if len_src == num_bpe_tokens[i][0] - 1:
                    source_lengths[i] = num_bpe_tokens[i][0] - 1
                else:
                    source_lengths[i] = num_bpe_tokens[i][0]

                target_lengths[i] = len(hypo[i].split())

                source[i] = remove_bpe(source[i][0], bpe_symbol)
                target[i] = remove_bpe(target[i], bpe_symbol)
                hypo[i] = remove_bpe(hypo[i], bpe_symbol)

                score[i] = float(score[i][0])
                pos_score[i] = pos_score[i][0]

            else:
                len_tgt = len(hypo[i][0].split())
                # record length without <eos>
                if len_tgt == num_bpe_tokens[i][0] - 1:
                    target_lengths[i] = num_bpe_tokens[i][0] - 1
                else:
                    target_lengths[i] = num_bpe_tokens[i][0]

                source_lengths[i] = len(source[i].split())

                if right_to_left:
                    source[i] = remove_bpe(make_right_to_left(source[i]), bpe_symbol)
                    target[i] = remove_bpe(make_right_to_left(target[i]), bpe_symbol)
                    hypo[i] = remove_bpe(make_right_to_left(hypo[i][0]), bpe_symbol)
                    score[i] = float(score[i][0])
                    pos_score[i] = pos_score[i][0]
                else:
                    assert (
                        len(hypo[i]) == 1
                    ), "expected only one hypothesis per source sentence"
                    source[i] = remove_bpe(source[i], bpe_symbol)
                    target[i] = remove_bpe(target[i], bpe_symbol)
                    hypo[i] = remove_bpe(hypo[i][0], bpe_symbol)
                    score[i] = float(score[i][0])
                    pos_score[i] = pos_score[i][0]

        self.rescore_source = source
        self.rescore_hypo = hypo
        self.rescore_score = score
        self.rescore_target = target
        self.rescore_pos_score = pos_score
        self.backwards = backwards
        self.right_to_left = right_to_left
        self.target_lengths = target_lengths
        self.source_lengths = source_lengths
process output from rescoring
examples/noisychannel/rerank_utils.py