<gh_stars>1-10
import logging

from collections import namedtuple
from nltk import edit_distance
from nltk.translate.bleu_score import SmoothingFunction, corpus_bleu
from nltk.translate.nist_score import corpus_nist

from components.evaluator.base_evaluator import BaseEvaluator
from components.utils.io import readlines

logger = logging.getLogger('main')

Score = namedtuple('Score', ['bleu', 'nist', 'edist'])

class SynEvaluator(BaseEvaluator):

    def __init__(self, config_dict):
        super(SynEvaluator, self).__init__()

        self.config = config_dict

        self.train_losses = []
        self.dev_losses = []

        # three official metrics used in SR 2018
        self.bleu_scores = []
        self.nist_scores = []
        self.edist_scores = []

    def get_scores_to_save(self):
        scores = list(zip(self.bleu_scores,
                          self.nist_scores,
                          self.edist_scores,
                          self.train_losses,
                          self.dev_losses))

        return scores

    def external_metric_eval(self, ref_fn, pred_fn):

        """
        Run external evaluation script (provided by the E2E NLG org)

        :param ref_fn: reference filename
        :param pred_fn: prediction filename
        :return:
        """

        score = SynEvaluator.read_predictions_and_eval(ref_fn=ref_fn, hyp_fn=pred_fn)

        self.bleu_scores.append(score.bleu)
        self.nist_scores.append(score.nist)
        self.edist_scores.append(score.edist)

        SynEvaluator.output_scores(score)

        return score

    @staticmethod
    def compute_final_scores(gold_snts, pred_snts):
        logger.info('Final scores:')

        refs = [[g.lower().split()] for g in gold_snts]
        hyps = [p.lower().split() for p in pred_snts]
        scores = SynEvaluator.compute_metric_scores(refs, hyps)
        SynEvaluator.output_scores(scores)

    @staticmethod
    def compute_metric_scores(refs, hyps):
        assert type(refs) == type(hyps) == list
        chencherry = SmoothingFunction()

        bl = corpus_bleu(refs, hyps, smoothing_function=chencherry.method2)
        ni = corpus_nist(refs, hyps, n=4)
        ed = compute_edit_distance(refs, hyps)

        return Score(bleu=bl, nist=ni, edist=ed)

    @staticmethod
    def read_predictions_and_eval(ref_fn, hyp_fn, normalise=True):
        logger.debug('Computing Syn scores:')
        logger.debug('\tReferences file: %s', ref_fn)
        logger.debug('\tHypotheses file: %s', hyp_fn)

        ref_strings = readlines(ref_fn)
        hyp_strings = readlines(hyp_fn)
        refs, hyps = get_ref_hyp_lists(ref_strings, hyp_strings, normalise)
        assert len(refs[0]) == 1, logger.error('Consider implementing multi-reference scorer!')

        score = SynEvaluator.compute_metric_scores(refs, hyps)

        return score

    @staticmethod
    def output_scores(scores):
        logger.info('BLEU = %0.3f NIST = %0.6f, EDIST = %0.3f', scores.bleu, scores.nist, scores.edist)

def compute_edit_distance(refs, hyps):

    total_len = 0.0
    edi = 0.0
    for r, h in zip(refs, hyps):
        reference_snt = ' '.join(r[0])
        hypothesis_snt = ' '.join(h)
        total_len += max(len(reference_snt), len(hypothesis_snt))
        edi += edit_distance(reference_snt, hypothesis_snt)

    edist = 1 - edi / total_len
    logger.info('[Unofficial edist: %0.4f]', edist)

    total_len = 0.0
    edi = 0.0
    for r, h in zip(refs, hyps):
        total_len += max(len(r[0]), len(h))
        edi += edit_distance(r[0], h)

    return (1 - edi / total_len)


def get_ref_hyp_lists(ref_strings, hyp_strings, normalise):

    if normalise:
        refs = [[s.lower().split()] for s in ref_strings]
        hyps = [s.lower().split() for s in hyp_strings]

    else:
        refs = [[s.split()] for s in ref_strings]
        hyps = [s.split() for s in hyp_strings]

    return (refs, hyps)

component = SynEvaluator








import logging

from collections import namedtuple
from nltk import edit_distance
from nltk.translate.bleu_score import SmoothingFunction, corpus_bleu
from nltk.translate.nist_score import corpus_nist

from components.evaluator.base_evaluator import BaseEvaluator
from components.utils.io import readlines

logger = logging.getLogger('main')

Score = namedtuple('Score', ['bleu', 'nist', 'edist'])

class SynEvaluator(BaseEvaluator):

    def __init__(self, config_dict):
        super(SynEvaluator, self).__init__()

        self.config = config_dict

        self.train_losses = []
        self.dev_losses = []

        # three official metrics used in SR 2018
        self.bleu_scores = []
        self.nist_scores = []
        self.edist_scores = []

    def get_scores_to_save(self):
        scores = list(zip(self.bleu_scores,
                          self.nist_scores,
                          self.edist_scores,
                          self.train_losses,
                          self.dev_losses))

        return scores

    def external_metric_eval(self, ref_fn, pred_fn):

        """
        Run external evaluation script (provided by the E2E NLG org)

        :param ref_fn: reference filename
        :param pred_fn: prediction filename
        :return:
        """

        score = SynEvaluator.read_predictions_and_eval(ref_fn=ref_fn, hyp_fn=pred_fn)

        self.bleu_scores.append(score.bleu)
        self.nist_scores.append(score.nist)
        self.edist_scores.append(score.edist)

        SynEvaluator.output_scores(score)

        return score

    @staticmethod
    def compute_final_scores(gold_snts, pred_snts):
        logger.info('Final scores:')

        refs = [[g.lower().split()] for g in gold_snts]
        hyps = [p.lower().split() for p in pred_snts]
        scores = SynEvaluator.compute_metric_scores(refs, hyps)
        SynEvaluator.output_scores(scores)

    @staticmethod
    def compute_metric_scores(refs, hyps):
        assert type(refs) == type(hyps) == list
        chencherry = SmoothingFunction()

        bl = corpus_bleu(refs, hyps, smoothing_function=chencherry.method2)
        ni = corpus_nist(refs, hyps, n=4)
        ed = compute_edit_distance(refs, hyps)

        return Score(bleu=bl, nist=ni, edist=ed)

    @staticmethod
    def read_predictions_and_eval(ref_fn, hyp_fn, normalise=True):
        logger.debug('Computing Syn scores:')
        logger.debug('\tReferences file: %s', ref_fn)
        logger.debug('\tHypotheses file: %s', hyp_fn)

        ref_strings = readlines(ref_fn)
        hyp_strings = readlines(hyp_fn)
        refs, hyps = get_ref_hyp_lists(ref_strings, hyp_strings, normalise)
        assert len(refs[0]) == 1, logger.error('Consider implementing multi-reference scorer!')

        score = SynEvaluator.compute_metric_scores(refs, hyps)

        return score

    @staticmethod
    def output_scores(scores):
        logger.info('BLEU = %0.3f NIST = %0.6f, EDIST = %0.3f', scores.bleu, scores.nist, scores.edist)

def compute_edit_distance(refs, hyps):

    total_len = 0.0
    edi = 0.0
    for r, h in zip(refs, hyps):
        reference_snt = ' '.join(r[0])
        hypothesis_snt = ' '.join(h)
        total_len += max(len(reference_snt), len(hypothesis_snt))
        edi += edit_distance(reference_snt, hypothesis_snt)

    edist = 1 - edi / total_len
    logger.info('[Unofficial edist: %0.4f]', edist)

    total_len = 0.0
    edi = 0.0
    for r, h in zip(refs, hyps):
        total_len += max(len(r[0]), len(h))
        edi += edit_distance(r[0], h)

    return (1 - edi / total_len)


def get_ref_hyp_lists(ref_strings, hyp_strings, normalise):

    if normalise:
        refs = [[s.lower().split()] for s in ref_strings]
        hyps = [s.lower().split() for s in hyp_strings]

    else:
        refs = [[s.split()] for s in ref_strings]
        hyps = [s.split() for s in hyp_strings]

    return (refs, hyps)

component = SynEvaluator
# three official metrics used in SR 2018 Run external evaluation script (provided by the E2E NLG org)          :param ref_fn: reference filename         :param pred_fn: prediction filename         :return: