# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import json
import spacy
import sys
import re
import kilt.kilt_utils as utils
from kilt.datasets.base_dataset import Dataset
from kilt import knowledge_source # remove later


class WizardDataset(Dataset):
    def __init__(self, name, input_file, output_file, log_file):
        super().__init__(name)
        self.input_file = "/private/home/edinan/ParlAI/data/wizard_of_wikipedia/train.json"
        self.output_file = output_file
        self.log_file = log_file
        self.nlp = spacy.load("en_core_web_sm")

    def get_chunks(self, num_chunks):
        with open(self.input_file, "r") as infile:
            all_data = json.load(infile)

        n = len(all_data)
        print("{} examples in the dataset".format(n))
        return utils.chunk_it(all_data, num_chunks)

    def process_chunk(self, chunk, ks, chunk_id=-1):
        missing_pages = 0.0
        short_exact_match = 0.0
        short_fuzzy_match = 0.0
        n = len(chunk)
        kilt_data = []

        for idx, datapoint in enumerate(chunk):

            print(
                "t: {}, p: {:.2f} %, mp: {:.1f}, exact: {:.1f}, fuzzy: {:.1f}".format(
                    chunk_id,
                    round(idx * 100 / n, 2),
                    missing_pages,
                    short_exact_match,
                    short_fuzzy_match,
                ),
                end="\r",
            )
            sys.stdout.flush()

            enumerate_id = 0

            episode_length = 0
            all_dialogue = datapoint['dialog']
            ongoing_dialogue = []
            for dialog in all_dialogue:
                ongoing_dialogue.append(dialog['text'])
                if 'checked_sentence' in dialog:
                    episode_length += 1
                    if dialog['checked_sentence'] != {}:
                        checked_sentence = dialog['checked_sentence']
                        # import pdb; pdb.set_trace();
                        topic = " ".join(list(checked_sentence.keys())[0].split("_")[1:-1])
                        print(topic)

                        page = ks.get_pages_by_title(topic)
                        if not page:
                            missing_pages += 1
                        else:
                            page = page[0]
                            kilt_record = {
                                # original data point id if available otherwise unique id
                                "id": enumerate_id,
                                # question / claim / sentence
                                # dialogue history goes here
                                "input": [turn for turn in ongoing_dialogue],
                            }
                            enumerate_id += 1

                            local_sem = 0.0
                            local_sfm = 0.0

                            chosen_answer = list(checked_sentence.values())[0]
                            answer_span = chosen_answer

                            (
                                paragraph_id,
                                start_character,
                                end_character,
                                bleu,
                            ) = utils.match_answer(
                                answer_span, page, nlp=self.nlp, debug=False
                            )

                            kilt_record_output = {
                                # answer in textual form
                                "answer": answer_span,
                                "provenance": [
                                    # list of relevant WikipediaPages / Spans as provenance for the answer from the ks
                                    {
                                        "wikipedia_id": page[
                                            "wikipedia_id"
                                        ],  # *mandatory* - ID Wikipedia Page
                                        "title": page[
                                            "wikipedia_title"
                                        ],  # *mandatory* - Title Wikipedia Page
                                        "start_paragraph_id": paragraph_id,  # start paragraph id with relevant info
                                        "start_character": start_character,
                                        "end_paragraph_id": paragraph_id,  # end paragraph id
                                        "end_character": end_character,
                                        "bleu_score": bleu,  # 1.0 when gold data is exactly matched, lower for fuzzy matches
                                    }
                                ],
                            }


                            if bleu == 1:
                                local_sem += 1
                            elif bleu < 1 and bleu >= 0:
                                local_sfm += 1
                            else:
                                print("ERROR: invalid bleu: {}".format(bleu))
                                sys.exit(-1)

                            # update kilt data
                            kilt_record["output"] = kilt_record_output
                            kilt_data.append(kilt_record)

        metadata = [missing_pages]
        return kilt_data, metadata

    def postprocess_metadata(self, metadata):
        missing_pages = 0.0
        short_exact_match = 0.0
        short_fuzzy_match = 0.0
        for met in metadata:
            if met == []:
                continue
            mp, sem, sfm = met
            missing_pages += mp
            short_exact_match += sem
            short_fuzzy_match += sfm

        print("Print stats")
        msg = "\n n: {:.1f}, missing pages: {:.1f}, short exact match: {:.1f}, short fuzzy match: {:.1f}".format(
            0, missing_pages, short_exact_match, short_fuzzy_match
        )
        print(msg)

        f = open(self.log_file, "w+")
        f.write(msg)
        f.close()

# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import json
import spacy
import sys
import re
import kilt.kilt_utils as utils
from kilt.datasets.base_dataset import Dataset
from kilt import knowledge_source # remove later


class WizardDataset(Dataset):
    def __init__(self, name, input_file, output_file, log_file):
        super().__init__(name)
        self.input_file = "/private/home/edinan/ParlAI/data/wizard_of_wikipedia/train.json"
        self.output_file = output_file
        self.log_file = log_file
        self.nlp = spacy.load("en_core_web_sm")

    def get_chunks(self, num_chunks):
        with open(self.input_file, "r") as infile:
            all_data = json.load(infile)

        n = len(all_data)
        print("{} examples in the dataset".format(n))
        return utils.chunk_it(all_data, num_chunks)

    def process_chunk(self, chunk, ks, chunk_id=-1):
        missing_pages = 0.0
        short_exact_match = 0.0
        short_fuzzy_match = 0.0
        n = len(chunk)
        kilt_data = []

        for idx, datapoint in enumerate(chunk):

            print(
                "t: {}, p: {:.2f} %, mp: {:.1f}, exact: {:.1f}, fuzzy: {:.1f}".format(
                    chunk_id,
                    round(idx * 100 / n, 2),
                    missing_pages,
                    short_exact_match,
                    short_fuzzy_match,
                ),
                end="\r",
            )
            sys.stdout.flush()

            enumerate_id = 0

            episode_length = 0
            all_dialogue = datapoint['dialog']
            ongoing_dialogue = []
            for dialog in all_dialogue:
                ongoing_dialogue.append(dialog['text'])
                if 'checked_sentence' in dialog:
                    episode_length += 1
                    if dialog['checked_sentence'] != {}:
                        checked_sentence = dialog['checked_sentence']
                        # import pdb; pdb.set_trace();
                        topic = " ".join(list(checked_sentence.keys())[0].split("_")[1:-1])
                        print(topic)

                        page = ks.get_pages_by_title(topic)
                        if not page:
                            missing_pages += 1
                        else:
                            page = page[0]
                            kilt_record = {
                                # original data point id if available otherwise unique id
                                "id": enumerate_id,
                                # question / claim / sentence
                                # dialogue history goes here
                                "input": [turn for turn in ongoing_dialogue],
                            }
                            enumerate_id += 1

                            local_sem = 0.0
                            local_sfm = 0.0

                            chosen_answer = list(checked_sentence.values())[0]
                            answer_span = chosen_answer

                            (
                                paragraph_id,
                                start_character,
                                end_character,
                                bleu,
                            ) = utils.match_answer(
                                answer_span, page, nlp=self.nlp, debug=False
                            )

                            kilt_record_output = {
                                # answer in textual form
                                "answer": answer_span,
                                "provenance": [
                                    # list of relevant WikipediaPages / Spans as provenance for the answer from the ks
                                    {
                                        "wikipedia_id": page[
                                            "wikipedia_id"
                                        ],  # *mandatory* - ID Wikipedia Page
                                        "title": page[
                                            "wikipedia_title"
                                        ],  # *mandatory* - Title Wikipedia Page
                                        "start_paragraph_id": paragraph_id,  # start paragraph id with relevant info
                                        "start_character": start_character,
                                        "end_paragraph_id": paragraph_id,  # end paragraph id
                                        "end_character": end_character,
                                        "bleu_score": bleu,  # 1.0 when gold data is exactly matched, lower for fuzzy matches
                                    }
                                ],
                            }


                            if bleu == 1:
                                local_sem += 1
                            elif bleu < 1 and bleu >= 0:
                                local_sfm += 1
                            else:
                                print("ERROR: invalid bleu: {}".format(bleu))
                                sys.exit(-1)

                            # update kilt data
                            kilt_record["output"] = kilt_record_output
                            kilt_data.append(kilt_record)

        metadata = [missing_pages]
        return kilt_data, metadata

    def postprocess_metadata(self, metadata):
        missing_pages = 0.0
        short_exact_match = 0.0
        short_fuzzy_match = 0.0
        for met in metadata:
            if met == []:
                continue
            mp, sem, sfm = met
            missing_pages += mp
            short_exact_match += sem
            short_fuzzy_match += sfm

        print("Print stats")
        msg = "\n n: {:.1f}, missing pages: {:.1f}, short exact match: {:.1f}, short fuzzy match: {:.1f}".format(
            0, missing_pages, short_exact_match, short_fuzzy_match
        )
        print(msg)

        f = open(self.log_file, "w+")
        f.write(msg)
        f.close()

# Copyright (c) Facebook, Inc. and its affiliates. # All rights reserved. # # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. # remove later # import pdb; pdb.set_trace(); # original data point id if available otherwise unique id # question / claim / sentence # dialogue history goes here # answer in textual form # list of relevant WikipediaPages / Spans as provenance for the answer from the ks # *mandatory* - ID Wikipedia Page # *mandatory* - Title Wikipedia Page # start paragraph id with relevant info # end paragraph id # 1.0 when gold data is exactly matched, lower for fuzzy matches # update kilt data