# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/8.5_codexplainer.d2v_vectorization.ipynb (unless otherwise specified).

__all__ = ['logger', 'check_file_existence', 'configure_dirs', 'Doc2VecVectorizer', 'Doc2VecVectorizerSP',
           'Doc2VecVectorizerHF']

# Cell

import numpy as np
import gensim
import pandas as pd
import os
import sentencepiece as spm


from tokenizers import Tokenizer
from abc import ABC, abstractmethod
from typing import Any, Optional

from datetime import datetime
from pathlib import Path

from ..mgmnt.prep.bpe_tokenization import HFTokenizer, SPTokenizer

# Cell

import logging

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
logger = logging.getLogger()

# Cell
# utils
def check_file_existence(path) -> bool:
    path = Path(path)
    if not path.exists():
        logging.error('Provided file cannot be found.')
        return False
    return True

# Cell

def configure_dirs(base_path: str, config_name: str, dataset_name: str) -> str:
    """
    Performs configuration of directories for storing vectors
    :param base_path:
    :param config_name:
    :param dataset_name:

    :return: Full configuration path
    """
    base_path = Path(base_path)
    base_path.mkdir(exist_ok=True)

    full_path = base_path / config_name
    full_path.mkdir(exist_ok=True)

    full_path = full_path / dataset_name
    full_path.mkdir(exist_ok=True)

    return str(full_path)

# Cell

class Doc2VecVectorizer(ABC):
    def __init__(self, tkzr_path:str, d2v_path: str, tokenizer: Optional[Any]=None):
        """
        Default constructor for Vectorizer class
        """
        self.tkzr_path = tkzr_path
        self.d2v_path = d2v_path

        self._load_doc2vec_model(d2v_path)
        if tokenizer is None:
            self._load_tokenizer_model(self.tkzr_path)
        else:
            self.tokenizer = tokenizer

    def tokenize_df(self, df: pd.DataFrame, code_column: str) -> pd.DataFrame:
        """
        Performs tokenization of a Dataframe

        :param df: DataFrame containing code
        :param code_column: Str indicating column name of code data

        :return: Tokenized DataFrame
        """

        return self.tokenizer.tokenize_df(df, code_column)

    @abstractmethod
    def _load_tokenizer_model(self, model_path: str):
        pass

    def _load_doc2vec_model(self, model_path: str):
        """
        :param model_path: Path to the model file
        :return: Gensim Doc2Vec model (corresponding to the loaded model)
        """
        if not check_file_existence(model_path):
            msg = 'Doc2vec model could no be loaded'
            logging.error('Doc2vec model could no be loaded')
            raise Exception(msg)

        model = gensim.models.Doc2Vec.load(model_path)
        self.d2v_model = model

    def infer_d2v(self, df: pd.DataFrame, tokenized_column: str, out_path: str,
                  config_name: str, sample_set_name: str,
                  perform_tokenization: Optional[bool]=False,
                  steps: Optional[int]=200) -> tuple:
        """
        Performs vectorization via Doc2Vec model
        :param df: Pandas DataFrame containing source code
        :param tokenized_column: Column name of the column corresponding to source code tokenized
                                 with the appropriate implementation
        :param out_path: String indicating the base location for storing vectors
        :param config_name: String indicating the model from which the samples came from
        :param sample_set_name: String indicating the base name for identifying the set of
                                 samples being processed
        :param perform_tokenization: Bool indicating whether tokenization is required or not
                                     (input df is previously tokenized or not)
        :param steps: Steps for the doc2vec infere
        :return: Tuple containing (idx of the input DF, obtained vectors)
        """

        tokenized_df = df.copy()

        if perform_tokenization:
            tokenized_df[tokenized_column] = self.tokenizer.tokenize_df(tokenized_df, 'code')

        inferred_vecs = np.array([self.d2v_model.infer_vector(tok_snippet, steps=200) \
                                  for tok_snippet in tokenized_df[tokenized_column].values])

        indices = np.array(df.index)

        dest_path = configure_dirs(out_path, config_name, sample_set_name)

        now = datetime.now()
        ts = str(datetime.timestamp(now))

        file_name = f"{dest_path}/{self.tok_name}-{ts}"

        np.save(f"{file_name}-idx", indices)
        np.save(f"{file_name}-ft_vecs", inferred_vecs)

        return indices, inferred_vecs

# Cell

class Doc2VecVectorizerSP(Doc2VecVectorizer):
    """
    Class to perform vectorization via Doc2Vec model
    leveraging SentencePiece to tokenizer sequences.
    """
    def __init__(self, sp_path: str, d2v_path: str, tokenizer: Optional[Any]=None):
        """
        :param sp_path: Path to the SentencePiece saved model
        :param d2v_path: Path to the Doc2Vec saved model
        """

        super().__init__(sp_path, d2v_path, tokenizer)
        self.tok_name = "sp"

    def _load_tokenizer_model(self, model_path: str):
        """
        Loads the sentence piece model stored in the specified path
        :param model_path: Path to the model file
        :return: SentencePieceProcessor object (corresponding to loaded model)
        """
        if not check_file_existence(model_path):
            msg = 'Sentence piece model could no be loaded'
            logging.error(msg)
            raise Exception(msg)

        sp_processor = spm.SentencePieceProcessor()
        sp_processor.load(model_path)
        self.tokenizer = sp_processor

# Cell

class Doc2VecVectorizerHF(Doc2VecVectorizer):
    """
    Class to perform vectorization via Doc2Vec model
    leveraging HF's Tokenizer
    """
    def __init__(self, tkzr_path: str, d2v_path: str, tokenizer: Optional[Any]=None):
        """
        :param tkzr_path: Path to the HF Tokenizer saved model
        :param d2v_path: Path to the Doc2Vec saved model
        """
        super().__init__(tkzr_path, d2v_path, tokenizer)
        self.tok_name = "hf"

    def _load_tokenizer_model(self, path: str) -> Tokenizer:
        """
        Function to load a saved HuggingFace tokenizer

        :param path: Path containing the tokenizer file
        :return:
        """

        if not check_file_existence(path):
            msg = 'HuggingFace tokenizer could no be loaded.'
            logging.error(msg)
            raise Exception(msg)

        self.tokenizer = Tokenizer.from_file(path)
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/8.5_codexplainer.d2v_vectorization.ipynb (unless otherwise specified).

__all__ = ['logger', 'check_file_existence', 'configure_dirs', 'Doc2VecVectorizer', 'Doc2VecVectorizerSP',
           'Doc2VecVectorizerHF']

# Cell

import numpy as np
import gensim
import pandas as pd
import os
import sentencepiece as spm


from tokenizers import Tokenizer
from abc import ABC, abstractmethod
from typing import Any, Optional

from datetime import datetime
from pathlib import Path

from ..mgmnt.prep.bpe_tokenization import HFTokenizer, SPTokenizer

# Cell

import logging

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
logger = logging.getLogger()

# Cell
# utils
def check_file_existence(path) -> bool:
    path = Path(path)
    if not path.exists():
        logging.error('Provided file cannot be found.')
        return False
    return True

# Cell

def configure_dirs(base_path: str, config_name: str, dataset_name: str) -> str:
    """
    Performs configuration of directories for storing vectors
    :param base_path:
    :param config_name:
    :param dataset_name:

    :return: Full configuration path
    """
    base_path = Path(base_path)
    base_path.mkdir(exist_ok=True)

    full_path = base_path / config_name
    full_path.mkdir(exist_ok=True)

    full_path = full_path / dataset_name
    full_path.mkdir(exist_ok=True)

    return str(full_path)

# Cell

class Doc2VecVectorizer(ABC):
    def __init__(self, tkzr_path:str, d2v_path: str, tokenizer: Optional[Any]=None):
        """
        Default constructor for Vectorizer class
        """
        self.tkzr_path = tkzr_path
        self.d2v_path = d2v_path

        self._load_doc2vec_model(d2v_path)
        if tokenizer is None:
            self._load_tokenizer_model(self.tkzr_path)
        else:
            self.tokenizer = tokenizer

    def tokenize_df(self, df: pd.DataFrame, code_column: str) -> pd.DataFrame:
        """
        Performs tokenization of a Dataframe

        :param df: DataFrame containing code
        :param code_column: Str indicating column name of code data

        :return: Tokenized DataFrame
        """

        return self.tokenizer.tokenize_df(df, code_column)

    @abstractmethod
    def _load_tokenizer_model(self, model_path: str):
        pass

    def _load_doc2vec_model(self, model_path: str):
        """
        :param model_path: Path to the model file
        :return: Gensim Doc2Vec model (corresponding to the loaded model)
        """
        if not check_file_existence(model_path):
            msg = 'Doc2vec model could no be loaded'
            logging.error('Doc2vec model could no be loaded')
            raise Exception(msg)

        model = gensim.models.Doc2Vec.load(model_path)
        self.d2v_model = model

    def infer_d2v(self, df: pd.DataFrame, tokenized_column: str, out_path: str,
                  config_name: str, sample_set_name: str,
                  perform_tokenization: Optional[bool]=False,
                  steps: Optional[int]=200) -> tuple:
        """
        Performs vectorization via Doc2Vec model
        :param df: Pandas DataFrame containing source code
        :param tokenized_column: Column name of the column corresponding to source code tokenized
                                 with the appropriate implementation
        :param out_path: String indicating the base location for storing vectors
        :param config_name: String indicating the model from which the samples came from
        :param sample_set_name: String indicating the base name for identifying the set of
                                 samples being processed
        :param perform_tokenization: Bool indicating whether tokenization is required or not
                                     (input df is previously tokenized or not)
        :param steps: Steps for the doc2vec infere
        :return: Tuple containing (idx of the input DF, obtained vectors)
        """

        tokenized_df = df.copy()

        if perform_tokenization:
            tokenized_df[tokenized_column] = self.tokenizer.tokenize_df(tokenized_df, 'code')

        inferred_vecs = np.array([self.d2v_model.infer_vector(tok_snippet, steps=200) \
                                  for tok_snippet in tokenized_df[tokenized_column].values])

        indices = np.array(df.index)

        dest_path = configure_dirs(out_path, config_name, sample_set_name)

        now = datetime.now()
        ts = str(datetime.timestamp(now))

        file_name = f"{dest_path}/{self.tok_name}-{ts}"

        np.save(f"{file_name}-idx", indices)
        np.save(f"{file_name}-ft_vecs", inferred_vecs)

        return indices, inferred_vecs

# Cell

class Doc2VecVectorizerSP(Doc2VecVectorizer):
    """
    Class to perform vectorization via Doc2Vec model
    leveraging SentencePiece to tokenizer sequences.
    """
    def __init__(self, sp_path: str, d2v_path: str, tokenizer: Optional[Any]=None):
        """
        :param sp_path: Path to the SentencePiece saved model
        :param d2v_path: Path to the Doc2Vec saved model
        """

        super().__init__(sp_path, d2v_path, tokenizer)
        self.tok_name = "sp"

    def _load_tokenizer_model(self, model_path: str):
        """
        Loads the sentence piece model stored in the specified path
        :param model_path: Path to the model file
        :return: SentencePieceProcessor object (corresponding to loaded model)
        """
        if not check_file_existence(model_path):
            msg = 'Sentence piece model could no be loaded'
            logging.error(msg)
            raise Exception(msg)

        sp_processor = spm.SentencePieceProcessor()
        sp_processor.load(model_path)
        self.tokenizer = sp_processor

# Cell

class Doc2VecVectorizerHF(Doc2VecVectorizer):
    """
    Class to perform vectorization via Doc2Vec model
    leveraging HF's Tokenizer
    """
    def __init__(self, tkzr_path: str, d2v_path: str, tokenizer: Optional[Any]=None):
        """
        :param tkzr_path: Path to the HF Tokenizer saved model
        :param d2v_path: Path to the Doc2Vec saved model
        """
        super().__init__(tkzr_path, d2v_path, tokenizer)
        self.tok_name = "hf"

    def _load_tokenizer_model(self, path: str) -> Tokenizer:
        """
        Function to load a saved HuggingFace tokenizer

        :param path: Path containing the tokenizer file
        :return:
        """

        if not check_file_existence(path):
            msg = 'HuggingFace tokenizer could no be loaded.'
            logging.error(msg)
            raise Exception(msg)

        self.tokenizer = Tokenizer.from_file(path)
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/8.5_codexplainer.d2v_vectorization.ipynb (unless otherwise specified). # Cell # Cell # Cell # utils # Cell Performs configuration of directories for storing vectors     :param base_path:     :param config_name:     :param dataset_name:      :return: Full configuration path # Cell Default constructor for Vectorizer class Performs tokenization of a Dataframe          :param df: DataFrame containing code         :param code_column: Str indicating column name of code data          :return: Tokenized DataFrame :param model_path: Path to the model file         :return: Gensim Doc2Vec model (corresponding to the loaded model) Performs vectorization via Doc2Vec model         :param df: Pandas DataFrame containing source code         :param tokenized_column: Column name of the column corresponding to source code tokenized                                  with the appropriate implementation         :param out_path: String indicating the base location for storing vectors         :param config_name: String indicating the model from which the samples came from         :param sample_set_name: String indicating the base name for identifying the set of                                  samples being processed         :param perform_tokenization: Bool indicating whether tokenization is required or not                                      (input df is previously tokenized or not)         :param steps: Steps for the doc2vec infere         :return: Tuple containing (idx of the input DF, obtained vectors) # Cell Class to perform vectorization via Doc2Vec model     leveraging SentencePiece to tokenizer sequences. :param sp_path: Path to the SentencePiece saved model         :param d2v_path: Path to the Doc2Vec saved model Loads the sentence piece model stored in the specified path         :param model_path: Path to the model file         :return: SentencePieceProcessor object (corresponding to loaded model) # Cell Class to perform vectorization via Doc2Vec model     leveraging HF's Tokenizer :param tkzr_path: Path to the HF Tokenizer saved model         :param d2v_path: Path to the Doc2Vec saved model Function to load a saved HuggingFace tokenizer          :param path: Path containing the tokenizer file         :return: