Optimus/code/examples/big_ae/eval_dialog_response.py

import numpy as np
import torch
import torch.nn.functional as F
from nltk.translate.bleu_score import sentence_bleu
from nltk.translate.bleu_score import SmoothingFunction
from sklearn.metrics.pairwise import cosine_similarity as cosine
from collections import Counter
import os, pickle

class Metrics:
    # based on https://raw.githubusercontent.com/guxd/DialogWAE/29f206af05bfe5fe28fec4448e208310a7c9258d/experiments/metrics.py
    
    def __init__(self, path_word2vec='../data/datasets/dailydialog_data/glove.twitter.27B.200d.txt'):
        """
        :param word2vec - a numpy array of word2vec with shape [vocab_size x emb_size]
        """
        self.path_word2vec = path_word2vec
        super(Metrics, self).__init__()
        self.load_word2vec(path_word2vec)

    def load_word2vec(self, path_word2vec):
        path_pkl = path_word2vec + '.pkl'
        if os.path.exists(path_pkl):
            print('loading word2vec from '+path_pkl)
            self.word2vec = pickle.load(open(path_pkl, 'rb'))
        else:
            self.word2vec = dict()
            for i, line in enumerate(open(path_word2vec, encoding='utf-8')):
                ss = line.strip('\n').split() 
                self.word2vec[ss[0]] = [float(v) for v in ss[1:]]
                if i % 1e4 == 0:
                    print('processed %ik word2vec'%(i/1e3))
            print('dumping word2vec to '+path_pkl)
            pickle.dump(self.word2vec, open(path_pkl, 'wb'))
        # pdb.set_trace()
        self.embed_dim = len(self.word2vec["."]) # len(self.word2vec.values()[0])
        print('loaded %i word2vec of dim %i'%(len(self.word2vec), self.embed_dim))

    def embedding(self, seqs): 
        # note: different from original implementation
        batch_size, seqlen = seqs.shape
        embs = np.zeros([batch_size, seqlen, self.embed_dim])
        for i in range(batch_size):
            for j in range(seqlen):
                w = seqs[i,j] 
                if w != '' and w in self.word2vec:
                    embs[i, j, :] = self.word2vec[w]
        return embs

    
    def extrema(self, embs, lens): # embs: [batch_size x seq_len x emb_size]  lens: [batch_size]
        """
        computes the value of every single dimension in the word vectors which has the greatest
        difference from zero.
        :param seq: sequence
        :param seqlen: length of sequence
        """
        # Find minimum and maximum value for every dimension in predictions
        batch_size, seq_len, emb_size = embs.shape
        max_mask = np.zeros((batch_size, seq_len, emb_size), dtype=np.int)
        for i,length in enumerate(lens):
            max_mask[i,:length,:]=1
        min_mask = 1-max_mask
        seq_max = (embs*max_mask).max(1) # [batch_sz x emb_sz]
        seq_min = (embs+min_mask).min(1)
        # Find the maximum absolute value in min and max data
        comp_mask = seq_max >= np.abs(seq_min)# [batch_sz x emb_sz]
        # Add vectors for finding final sequence representation for predictions
        extrema_emb = seq_max* comp_mask + seq_min* np.logical_not(comp_mask)
        return extrema_emb
    
    def mean(self, embs, lens):
        batch_size, seq_len, emb_size=embs.shape
        mask = np.zeros((batch_size, seq_len, emb_size), dtype=np.int)
        for i,length in enumerate(lens):
            mask[i,:length,:]=1
        return (embs*mask).sum(1)/(mask.sum(1)+1e-8)

    def sim_bleu(self, hyps, ref):
        """
        :param ref - a list of tokens of the reference
        :param hyps - a list of tokens of the hypothesis
    
        :return maxbleu - recall bleu
        :return avgbleu - precision bleu
        """
        scores = []
        for hyp in hyps:
            try:
                scores.append(sentence_bleu([ref], hyp, smoothing_function=SmoothingFunction().method7,
                                        weights=[1./3, 1./3, 1./3]))
            except:
                scores.append(0.0)
        return np.max(scores), np.mean(scores)


    def sim_bow(self, pred, pred_lens, ref, ref_lens):
        """
        :param pred - ndarray [batch_size x seqlen]
        :param pred_lens - list of integers
        :param ref - ndarray [batch_size x seqlen]
        """
        # look up word embeddings for prediction and reference
        emb_pred = self.embedding(pred) # [batch_sz x seqlen1 x emb_sz]
        emb_ref = self.embedding(ref) # [batch_sz x seqlen2 x emb_sz]
        
        ext_emb_pred=self.extrema(emb_pred, pred_lens)
        ext_emb_ref=self.extrema(emb_ref, ref_lens)
        bow_extrema=cosine(ext_emb_pred, ext_emb_ref) # [batch_sz_pred x batch_sz_ref]
        
        avg_emb_pred = self.mean(emb_pred, pred_lens) # Calculate mean over seq
        avg_emb_ref = self.mean(emb_ref, ref_lens) 
        bow_avg = cosine(avg_emb_pred, avg_emb_ref) # [batch_sz_pred x batch_sz_ref]

        
        batch_pred, seqlen_pred, emb_size=emb_pred.shape
        batch_ref, seqlen_ref, emb_size=emb_ref.shape
        cos_sim = cosine(emb_pred.reshape((-1, emb_size)), emb_ref.reshape((-1, emb_size))) # [(batch_sz*seqlen1)x(batch_sz*seqlen2)]
        cos_sim = cos_sim.reshape((batch_pred, seqlen_pred, batch_ref, seqlen_ref))
        # Find words with max cosine similarity
        max12 = cos_sim.max(1).mean(2) # max over seqlen_pred
        max21 = cos_sim.max(3).mean(1) # max over seqlen_ref
        bow_greedy=(max12+max21)/2 # [batch_pred x batch_ref(1)]
        return np.max(bow_extrema), np.max(bow_avg), np.max(bow_greedy)
    
    def div_distinct(self, seqs, seq_lens):
        """
        distinct-1 distinct-2 metrics for diversity measure proposed 
        by Li et al. "A Diversity-Promoting Objective Function for Neural Conversation Models"
        we counted numbers of distinct unigrams and bigrams in the generated responses 
        and divide the numbers by total number of unigrams and bigrams. 
        The two metrics measure how informative and diverse the generated responses are. 
        High numbers and high ratios mean that there is much content in the generated responses, 
        and high numbers further indicate that the generated responses are long
        """
        batch_size = seqs.shape[0]
        intra_dist1, intra_dist2=np.zeros(batch_size), np.zeros(batch_size)
        
        n_unigrams, n_bigrams, n_unigrams_total , n_bigrams_total = 0. ,0., 0., 0.
        unigrams_all, bigrams_all = Counter(), Counter()
        for b in range(batch_size):
            unigrams= Counter([tuple(seqs[b,i:i+1]) for i in range(seq_lens[b])])
            bigrams = Counter([tuple(seqs[b,i:i+2]) for i in range(seq_lens[b]-1)])
            intra_dist1[b]=(len(unigrams.items())+1e-12)/(seq_lens[b]+1e-5)
            intra_dist2[b]=(len(bigrams.items())+1e-12)/(max(0, seq_lens[b]-1)+1e-5)
            
            unigrams_all.update([tuple(seqs[b,i:i+1]) for i in range(seq_lens[b])])
            bigrams_all.update([tuple(seqs[b,i:i+2]) for i in range(seq_lens[b]-1)])
            n_unigrams_total += seq_lens[b]
            n_bigrams_total += max(0, seq_lens[b]-1)

        inter_dist1 = (len(unigrams_all.items())+1e-12)/(n_unigrams_total+1e-5)
        inter_dist2 = (len(bigrams_all.items())+1e-12)/(n_bigrams_total+1e-5)
        return intra_dist1, intra_dist2, inter_dist1, inter_dist2

import pdb

def eval_dialog_response(generated_text_file_path):
    """
    based on: https://github.com/guxd/DialogWAE/blob/29f206af05bfe5fe28fec4448e208310a7c9258d/sample.py
    quoted from the DialogWAE paper: https://arxiv.org/pdf/1805.12352.pdf
    * "For each test context, we sample 10 responses from the models and compute their BLEU scores"
    * "We use Glove vectors" "For each test context, we report the maximum BOW embedding score among the 10 sampled responses."
    * "intra-dist as the average of distinct values within each sampled response"
    " "inter-dist as the distinct value among all sampled responses."
    """
    metrics = Metrics()
    d_ref = dict()
    d_hyp = dict()
    for line in open(generated_text_file_path, encoding='utf-8'):
        line = line.strip('\n').strip()
        if len(line) == 0:
            continue
        src, ref, hyp = line.split('\t')
        src = src.strip()
        ref = ref.strip().split()
        hyp = hyp.strip().split()
        if src not in d_ref:
            d_ref[src] = ref
            d_hyp[src] = [hyp]
        else:
            d_hyp[src].append(hyp)
    
    n = len(d_ref)
    print(generated_text_file_path)
    print('n_src\t%i'%n)

    avg_lens = 0
    maxbleu = 0
    avgbleu = 0
    intra_dist1, intra_dist2, inter_dist1, inter_dist2 = 0,0,0,0
    bow_extrema, bow_avg, bow_greedy = 0,0,0
    for src in d_ref:
        m, a = metrics.sim_bleu(d_hyp[src], d_ref[src])
        maxbleu += m
        avgbleu += a
        
        seq_len = [len(hyp) for hyp in d_hyp[src]]
        max_len = max(seq_len)
        seqs = []
        for hyp in d_hyp[src]:
            padded = hyp + [''] * (max_len - len(hyp))
            seqs.append(np.reshape(padded, [1, -1]))
        seqs = np.concatenate(seqs, axis=0)
        intra1, intra2, inter1, inter2 = metrics.div_distinct(seqs, seq_len)
        intra_dist1 += np.mean(intra1)
        intra_dist2 += np.mean(intra2)
        inter_dist1 += inter1
        inter_dist2 += inter2
        
        n_hyp = len(d_hyp[src])
        seqs_ref = np.concatenate([np.reshape(d_ref[src], [1,-1])] * n_hyp, axis=0)
        seq_len_ref = [len(d_ref[src])] * n_hyp
        if metrics.word2vec is not None:
            extrema, avg, greedy = metrics.sim_bow(seqs, seq_len, seqs_ref, seq_len_ref)
            bow_extrema += extrema
            bow_avg += avg
            bow_greedy += greedy

        avg_lens += np.mean(seq_len)

    recall_bleu = maxbleu/n
    prec_bleu = avgbleu/n
    f1 = 2*(prec_bleu*recall_bleu) / (prec_bleu+recall_bleu+10e-12)
    
    print('BLEU')
    print('  R\t%.3f'%recall_bleu)
    print('  P\t%.3f'%prec_bleu)
    print('  F1\t%.3f'%f1)
    print('BOW')
    print('  A\t%.3f'%(bow_avg/n))
    print('  E\t%.3f'%(bow_extrema/n))
    print('  G\t%.3f'%(bow_greedy/n))
    print('intra_dist')
    print('  1\t%.3f'%(intra_dist1/n))
    print('  2\t%.3f'%(intra_dist2/n))
    print('inter_dist')
    print('  1\t%.3f'%(inter_dist1/n))
    print('  2\t%.3f'%(inter_dist2/n))
    print('avg_L\t%.1f'%(avg_lens/n))

    results = {
        "BLEU_R": recall_bleu, "BLEU_P": prec_bleu, "BLEU_F1": f1, "BOW_A": bow_avg/n, "BOW_E": bow_extrema/n, "BOW_G": bow_greedy/n, "intra_dist1": intra_dist1/n, "intra_dist2": intra_dist2/n, "inter_dist1": inter_dist1/n, "inter_dist2": inter_dist2/n, "avg_L": avg_lens/n
    }

    return results



def create_rand_baseline():
    path = 'data/datasets/dailydialog_data/test.txt'
    srcs = []
    refs = []
    for line in open(path, encoding='utf-8'):
        src, ref = line.strip('\n').split('\t')
        srcs.append(src.strip())
        refs.append(ref.strip())
    
    hyps = set()
    path = 'data/datasets/dailydialog_data/train.txt'
    for line in open(path, encoding='utf-8'):
        _, ref = line.strip('\n').split('\t')
        hyps.add(ref)
        if len(hyps) == len(srcs) *10:
            print('collected training ref')
            break
    
    hyps = list(hyps)
    lines = []
    j = 0
    for i in range(len(srcs)):
        lines += ['\t'.join([srcs[i], refs[i], hyp]) for hyp in hyps[j:j+10]]
        j = j + 10
    with open('out/rand.tsv', 'w', encoding='utf-8') as f:
        f.write('\n'.join(lines))


def create_human_baseline():
    path = 'data/datasets/dailydialog_data/test.txt'
    lines = []
    for line in open(path, encoding='utf-8'):
        src, ref = line.strip('\n').split('\t')
        src = src.strip()
        ref = ref.strip()
        lines.append('\t'.join([src, ref, ref]))
        
    with open('out/human.tsv', 'w', encoding='utf-8') as f:
        f.write('\n'.join(lines))


if __name__ == "__main__":
    #create_rand_baseline()
    #create_human_baseline()
    eval_dialog_response('out/eval_text_generation_results (1).txt')
    #eval('out/rand.tsv')