<filename>convlab2/policy/rule/crosswoz/evaluate.py
import json
import zipfile
from collections import Counter
from pprint import pprint
from convlab2.policy.rule.crosswoz.rule_simulator import Simulator
from copy import deepcopy


def read_zipped_json(filepath, filename):
    archive = zipfile.ZipFile(filepath, 'r')
    return json.load(archive.open(filename))


def calculateF1(predict_golden):
    TP, FP, FN = 0, 0, 0
    for item in predict_golden:
        predicts = item['predict']
        labels = item['golden']
        for quad in predicts:
            if quad in labels:
                TP += 1
            else:
                FP += 1
        for quad in labels:
            if quad not in predicts:
                FN += 1
    # print(TP, FP, FN)
    precision = 1.0 * TP / (TP + FP)
    recall = 1.0 * TP / (TP + FN)
    F1 = 2.0 * precision * recall / (precision + recall)
    return precision, recall, F1


def calculateJointState(predict_golden):
    res = []
    for item in predict_golden:
        predicts = item['predict']
        labels = item['golden']
        res.append(predicts==labels)
    return sum(res)/len(res)


def calculateSlotState(predict_golden):
    res = []
    for item in predict_golden:
        predicts = item['predict']
        labels = item['golden']
        for x, y in zip(predicts, labels):
            res.append(x==y)
    return sum(res) / len(res)


def begin_active_tuple_num(data):
    active_tuples = []
    for task_id, item in data.items():
        for i, turn in enumerate(item['messages']):
            active_t = []
            for j, st in enumerate(turn['user_state']):
                if st[-1]:
                    active_t.append(j)
            active_tuples.append(tuple(active_t))
            break
    c = Counter(active_tuples)
    s = sum(c.values())
    pprint({x[0]:x[1]/s for x in c.items()})


def begin_da_type(data):
    da_type = []
    for task_id, item in data.items():
        for i, turn in enumerate(item['messages']):
            for intent, domain, slot, value in turn['dialog_act']:
                if intent == 'General':
                    intent = '+'.join([intent,domain,slot,value])
                da_type.append(intent)
            break
    c = Counter(da_type)
    s = sum(c.values())
    print('da_type')
    pprint({x[0]: x[1] / s for x in c.items()})


def end_usr_da_type(data):
    da_type = []
    for task_id, item in data.items():
        for i, turn in enumerate(item['messages'][::-1]):
            if turn['role']=='sys':
                continue
            da_key = ''
            for intent, domain, slot, value in turn['dialog_act']:
                if intent == 'General':
                    intent = '+'.join([intent,domain,slot,value])
                da_key+=intent
                # da_type.append(intent)
            da_type.append(da_key)
            break
    c = Counter(da_type)
    s = sum(c.values())
    print('da_type')
    pprint({x[0]: x[1] / s for x in c.items()})


def eval_begin_da_predict(data):
    predict_golden = []
    for task_id, item in data.items():
        for i, turn in enumerate(item['messages']):
            active_t = []
            for j, st in enumerate(turn['user_state']):
                if st[-1]:
                    active_t.append(st)
            da = [['General', 'greet', 'none', 'none']]
            for intent, domain, slot, value, expressed in active_t:
                if not value:
                    da.append(['Request', domain, slot, ''])
                elif isinstance(value, str):
                    da.append(['Inform', domain, slot, value])
                elif isinstance(value, list):
                    for v in value:
                        da.append(['Inform', domain, slot, v])
                else:
                    assert 0
            predict_golden.append({'predict':sorted(da), 'golden': turn['dialog_act']})
            break
    print('First turn da prediction given state precision/recall/f1',calculateF1(predict_golden))


def eval_simulator_performance(data, goal_type=None):
    begin_da_predict_golden = []
    state_da_predict_golden = []
    state_predict_golden = []
    simulator = Simulator()
    for task_id, item in data.items():
        if goal_type and item['type']!=goal_type:
            continue
        for i, turn in enumerate(item['messages']):
            if turn['role']=='usr':
                if i==0:
                    simulator.init_session(goal=item['goal'])
                    begin_da_predict_golden.append({
                        'predict': simulator.begin_da(),
                        'golden': turn['dialog_act']
                    })
                else:
                    last_turn = item['messages'][i - 2]
                    usr_da = item['messages'][i - 2]['dialog_act']
                    sys_da = item['messages'][i - 1]['dialog_act']
                    simulator.init_session(goal=item['goal'], state=deepcopy(last_turn['user_state']))
                    simulator.state_update(prev_user_da=usr_da, prev_sys_da=sys_da)
                    cur_da = simulator.state_predict()
                    new_state = deepcopy(simulator.state)
                    state_da_predict_golden.append({
                        'predict': cur_da,
                        'golden': turn['dialog_act']
                    })
                    state_predict_golden.append({
                        'predict': new_state,
                        'golden': turn['user_state']
                    })

    print('begin da', calculateF1(begin_da_predict_golden))
    print('state da', calculateF1(state_da_predict_golden))
    print('all da', calculateF1(begin_da_predict_golden+state_da_predict_golden))
    print('joint state', calculateJointState(state_predict_golden))
    print('slot state', calculateSlotState(state_predict_golden))


def eval_state_predict(data):
    def state_update(prev_state, cur_state):
        update = []
        for prev_ele, cur_ele in zip(prev_state, cur_state):
            if cur_ele != prev_ele:
                update.append(cur_ele)
        id = 1
        for ele in cur_state[::-1]:
            if ele[-1]:
                id = ele[0]
                break
        return update, id

    simulator = Simulator()
    for task_id, item in data.items():
        for i, turn in enumerate(item['messages']):
            if turn['role']=='usr' and i > 0:
                last_turn = item['messages'][i-2]
                usr_da = item['messages'][i-2]['dialog_act']
                sys_da = item['messages'][i-1]['dialog_act']
                simulator.init_session(goal=item['goal'],state=deepcopy(last_turn['user_state']))
                simulator.state_update(prev_user_da=usr_da, prev_sys_da=sys_da)
                cur_da = simulator.state_predict()
                new_state = simulator.state
                # print('old state:')
                # pprint(last_turn['user_state'])
                # if 'NoOffer' in [x[0] for x in item['messages'][i-1]['dialog_act']]:
                print(item['messages'][i-2]['content'])
                print(item['messages'][i-1]['content'])
                print(turn['content'])
                print('usr da')
                pprint(usr_da)
                print('sys da')
                pprint(sys_da)
                print('predict state update:')
                pprint(state_update(last_turn['user_state'], new_state))
                print('golden state:')
                pprint(state_update(last_turn['user_state'], turn['user_state']))
                print('predict usr da')
                pprint(cur_da)
                print('golden usr da')
                pprint(turn['dialog_act'])
                print('-'*100)


if __name__ == '__main__':
    # train_data_path = '../../data/raw_data/train.json.zip'
    # train_data = read_zipped_json(train_data_path, 'train.json')
    # begin_active_tuple_num(train_data)
    # begin_da_type(train_data)
    # end_usr_da_type(train_data)
    # eval_begin_da_predict(train_data)
    # print(len(train_data))
    # eval_state_predict(train_data)
    test_data_path = '../../../../data/crosswoz/test.json.zip'
    test_data = read_zipped_json(test_data_path, 'test.json')
    for goal_type in ['单领域', '独立多领域', '独立多领域+交通', '不独立多领域', '不独立多领域+交通', None]:
        print(goal_type)
        eval_simulator_performance(test_data, goal_type=goal_type)

<filename>convlab2/policy/rule/crosswoz/evaluate.py
import json
import zipfile
from collections import Counter
from pprint import pprint
from convlab2.policy.rule.crosswoz.rule_simulator import Simulator
from copy import deepcopy


def read_zipped_json(filepath, filename):
    archive = zipfile.ZipFile(filepath, 'r')
    return json.load(archive.open(filename))


def calculateF1(predict_golden):
    TP, FP, FN = 0, 0, 0
    for item in predict_golden:
        predicts = item['predict']
        labels = item['golden']
        for quad in predicts:
            if quad in labels:
                TP += 1
            else:
                FP += 1
        for quad in labels:
            if quad not in predicts:
                FN += 1
    # print(TP, FP, FN)
    precision = 1.0 * TP / (TP + FP)
    recall = 1.0 * TP / (TP + FN)
    F1 = 2.0 * precision * recall / (precision + recall)
    return precision, recall, F1


def calculateJointState(predict_golden):
    res = []
    for item in predict_golden:
        predicts = item['predict']
        labels = item['golden']
        res.append(predicts==labels)
    return sum(res)/len(res)


def calculateSlotState(predict_golden):
    res = []
    for item in predict_golden:
        predicts = item['predict']
        labels = item['golden']
        for x, y in zip(predicts, labels):
            res.append(x==y)
    return sum(res) / len(res)


def begin_active_tuple_num(data):
    active_tuples = []
    for task_id, item in data.items():
        for i, turn in enumerate(item['messages']):
            active_t = []
            for j, st in enumerate(turn['user_state']):
                if st[-1]:
                    active_t.append(j)
            active_tuples.append(tuple(active_t))
            break
    c = Counter(active_tuples)
    s = sum(c.values())
    pprint({x[0]:x[1]/s for x in c.items()})


def begin_da_type(data):
    da_type = []
    for task_id, item in data.items():
        for i, turn in enumerate(item['messages']):
            for intent, domain, slot, value in turn['dialog_act']:
                if intent == 'General':
                    intent = '+'.join([intent,domain,slot,value])
                da_type.append(intent)
            break
    c = Counter(da_type)
    s = sum(c.values())
    print('da_type')
    pprint({x[0]: x[1] / s for x in c.items()})


def end_usr_da_type(data):
    da_type = []
    for task_id, item in data.items():
        for i, turn in enumerate(item['messages'][::-1]):
            if turn['role']=='sys':
                continue
            da_key = ''
            for intent, domain, slot, value in turn['dialog_act']:
                if intent == 'General':
                    intent = '+'.join([intent,domain,slot,value])
                da_key+=intent
                # da_type.append(intent)
            da_type.append(da_key)
            break
    c = Counter(da_type)
    s = sum(c.values())
    print('da_type')
    pprint({x[0]: x[1] / s for x in c.items()})


def eval_begin_da_predict(data):
    predict_golden = []
    for task_id, item in data.items():
        for i, turn in enumerate(item['messages']):
            active_t = []
            for j, st in enumerate(turn['user_state']):
                if st[-1]:
                    active_t.append(st)
            da = [['General', 'greet', 'none', 'none']]
            for intent, domain, slot, value, expressed in active_t:
                if not value:
                    da.append(['Request', domain, slot, ''])
                elif isinstance(value, str):
                    da.append(['Inform', domain, slot, value])
                elif isinstance(value, list):
                    for v in value:
                        da.append(['Inform', domain, slot, v])
                else:
                    assert 0
            predict_golden.append({'predict':sorted(da), 'golden': turn['dialog_act']})
            break
    print('First turn da prediction given state precision/recall/f1',calculateF1(predict_golden))


def eval_simulator_performance(data, goal_type=None):
    begin_da_predict_golden = []
    state_da_predict_golden = []
    state_predict_golden = []
    simulator = Simulator()
    for task_id, item in data.items():
        if goal_type and item['type']!=goal_type:
            continue
        for i, turn in enumerate(item['messages']):
            if turn['role']=='usr':
                if i==0:
                    simulator.init_session(goal=item['goal'])
                    begin_da_predict_golden.append({
                        'predict': simulator.begin_da(),
                        'golden': turn['dialog_act']
                    })
                else:
                    last_turn = item['messages'][i - 2]
                    usr_da = item['messages'][i - 2]['dialog_act']
                    sys_da = item['messages'][i - 1]['dialog_act']
                    simulator.init_session(goal=item['goal'], state=deepcopy(last_turn['user_state']))
                    simulator.state_update(prev_user_da=usr_da, prev_sys_da=sys_da)
                    cur_da = simulator.state_predict()
                    new_state = deepcopy(simulator.state)
                    state_da_predict_golden.append({
                        'predict': cur_da,
                        'golden': turn['dialog_act']
                    })
                    state_predict_golden.append({
                        'predict': new_state,
                        'golden': turn['user_state']
                    })

    print('begin da', calculateF1(begin_da_predict_golden))
    print('state da', calculateF1(state_da_predict_golden))
    print('all da', calculateF1(begin_da_predict_golden+state_da_predict_golden))
    print('joint state', calculateJointState(state_predict_golden))
    print('slot state', calculateSlotState(state_predict_golden))


def eval_state_predict(data):
    def state_update(prev_state, cur_state):
        update = []
        for prev_ele, cur_ele in zip(prev_state, cur_state):
            if cur_ele != prev_ele:
                update.append(cur_ele)
        id = 1
        for ele in cur_state[::-1]:
            if ele[-1]:
                id = ele[0]
                break
        return update, id

    simulator = Simulator()
    for task_id, item in data.items():
        for i, turn in enumerate(item['messages']):
            if turn['role']=='usr' and i > 0:
                last_turn = item['messages'][i-2]
                usr_da = item['messages'][i-2]['dialog_act']
                sys_da = item['messages'][i-1]['dialog_act']
                simulator.init_session(goal=item['goal'],state=deepcopy(last_turn['user_state']))
                simulator.state_update(prev_user_da=usr_da, prev_sys_da=sys_da)
                cur_da = simulator.state_predict()
                new_state = simulator.state
                # print('old state:')
                # pprint(last_turn['user_state'])
                # if 'NoOffer' in [x[0] for x in item['messages'][i-1]['dialog_act']]:
                print(item['messages'][i-2]['content'])
                print(item['messages'][i-1]['content'])
                print(turn['content'])
                print('usr da')
                pprint(usr_da)
                print('sys da')
                pprint(sys_da)
                print('predict state update:')
                pprint(state_update(last_turn['user_state'], new_state))
                print('golden state:')
                pprint(state_update(last_turn['user_state'], turn['user_state']))
                print('predict usr da')
                pprint(cur_da)
                print('golden usr da')
                pprint(turn['dialog_act'])
                print('-'*100)


if __name__ == '__main__':
    # train_data_path = '../../data/raw_data/train.json.zip'
    # train_data = read_zipped_json(train_data_path, 'train.json')
    # begin_active_tuple_num(train_data)
    # begin_da_type(train_data)
    # end_usr_da_type(train_data)
    # eval_begin_da_predict(train_data)
    # print(len(train_data))
    # eval_state_predict(train_data)
    test_data_path = '../../../../data/crosswoz/test.json.zip'
    test_data = read_zipped_json(test_data_path, 'test.json')
    for goal_type in ['单领域', '独立多领域', '独立多领域+交通', '不独立多领域', '不独立多领域+交通', None]:
        print(goal_type)
        eval_simulator_performance(test_data, goal_type=goal_type)

# print(TP, FP, FN) # da_type.append(intent) # print('old state:') # pprint(last_turn['user_state']) # if 'NoOffer' in [x[0] for x in item['messages'][i-1]['dialog_act']]: # train_data_path = '../../data/raw_data/train.json.zip' # train_data = read_zipped_json(train_data_path, 'train.json') # begin_active_tuple_num(train_data) # begin_da_type(train_data) # end_usr_da_type(train_data) # eval_begin_da_predict(train_data) # print(len(train_data)) # eval_state_predict(train_data)