<gh_stars>10-100
# -*- coding: utf-8 -*-
# @Time    : 2019-10-31 10:58
# @Author  : yingyuankai
# @Email   : <EMAIL>
# @File    : tokenizer_base.py

from abc import ABCMeta, abstractmethod
import logging
from typing import Dict, Optional, List

from aispace.utils.hparams import Hparams
from aispace.utils.registry import Registry

__all__ = [
    "BaseTokenizer"
]


SPECIAL_TOKENS_MAP_FILE = 'special_tokens_map.json'
ADDED_TOKENS_FILE = 'added_tokens.json'
CONFIG_FILE = 'config.json'

logger = logging.getLogger(__name__)


class BaseTokenizer(Registry):
    __metaclass__ = ABCMeta

    _MAX_INPUT_SIZE: Dict[str, Optional[int]]
    _VOCAB_FILE_NAMES: Dict[str, str]

    def __init__(self, hparams: Hparams):
        self._hparams = hparams
        self._vocab = None

    @abstractmethod
    def tokenize(self, input):
        """text to tokens

        :param input:
        :return:
        """
        raise NotImplementedError

    @abstractmethod
    def detokenizer(self, tokens: List[str]) -> str:
        r""" de + tokenizer, antonym of tokenizer.
        Maps a sequence of tokens (string) in a single string.
        The most simple way to do it is :python:`' '.join(tokens)`, but we
        often want to remove sub-word tokenization artifacts at the same time.
        """
        raise NotImplementedError

    @abstractmethod
    def encode(self,
               text_a,
               text_b=None,
               max_seq_length: Optional[int] = None):
        """text to idx

        :param text_a:
        :param text_b:
        :param max_seq_length:
        :return:
        """
        raise NotImplementedError

    @abstractmethod
    def decode(self, idx):
        """idx to text

        :param idx:
        :return:
        """
        raise NotImplementedError

    @property
    def vocab(self):
        return self._vocab

    @property
    def max_len(self):
        if not self._hparams.has_key("max_len"):
            logger.error("Must specify the max_len in tokenizer")
            return None
        return self._hparams.get("max_len")

    @staticmethod
    def clean_up_tokenization(out_string: str) -> str:
        r"""Clean up a list of simple English tokenization artifacts like
        spaces before punctuations and abbreviated forms.
        """
        out_string = out_string.replace(' .', '.').replace(' ?', '?'). \
            replace(' !', '!').replace(' ,', ',').replace(" ' ", "'"). \
            replace(" n't", "n't").replace(" 'm", "'m"). \
            replace(" do not", " don't").replace(" 's", "'s"). \
            replace(" 've", "'ve").replace(" 're", "'re")
        return out_string
# -*- coding: utf-8 -*-
# @Time    : 2019-10-31 10:58
# @Author  : yingyuankai
# @Email   : <EMAIL>
# @File    : tokenizer_base.py

from abc import ABCMeta, abstractmethod
import logging
from typing import Dict, Optional, List

from aispace.utils.hparams import Hparams
from aispace.utils.registry import Registry

__all__ = [
    "BaseTokenizer"
]


SPECIAL_TOKENS_MAP_FILE = 'special_tokens_map.json'
ADDED_TOKENS_FILE = 'added_tokens.json'
CONFIG_FILE = 'config.json'

logger = logging.getLogger(__name__)


class BaseTokenizer(Registry):
    __metaclass__ = ABCMeta

    _MAX_INPUT_SIZE: Dict[str, Optional[int]]
    _VOCAB_FILE_NAMES: Dict[str, str]

    def __init__(self, hparams: Hparams):
        self._hparams = hparams
        self._vocab = None

    @abstractmethod
    def tokenize(self, input):
        """text to tokens

        :param input:
        :return:
        """
        raise NotImplementedError

    @abstractmethod
    def detokenizer(self, tokens: List[str]) -> str:
        r""" de + tokenizer, antonym of tokenizer.
        Maps a sequence of tokens (string) in a single string.
        The most simple way to do it is :python:`' '.join(tokens)`, but we
        often want to remove sub-word tokenization artifacts at the same time.
        """
        raise NotImplementedError

    @abstractmethod
    def encode(self,
               text_a,
               text_b=None,
               max_seq_length: Optional[int] = None):
        """text to idx

        :param text_a:
        :param text_b:
        :param max_seq_length:
        :return:
        """
        raise NotImplementedError

    @abstractmethod
    def decode(self, idx):
        """idx to text

        :param idx:
        :return:
        """
        raise NotImplementedError

    @property
    def vocab(self):
        return self._vocab

    @property
    def max_len(self):
        if not self._hparams.has_key("max_len"):
            logger.error("Must specify the max_len in tokenizer")
            return None
        return self._hparams.get("max_len")

    @staticmethod
    def clean_up_tokenization(out_string: str) -> str:
        r"""Clean up a list of simple English tokenization artifacts like
        spaces before punctuations and abbreviated forms.
        """
        out_string = out_string.replace(' .', '.').replace(' ?', '?'). \
            replace(' !', '!').replace(' ,', ',').replace(" ' ", "'"). \
            replace(" n't", "n't").replace(" 'm", "'m"). \
            replace(" do not", " don't").replace(" 's", "'s"). \
            replace(" 've", "'ve").replace(" 're", "'re")
        return out_string
# -*- coding: utf-8 -*- # @Time    : 2019-10-31 10:58 # @Author  : yingyuankai # @Email   : <EMAIL> # @File    : tokenizer_base.py text to tokens          :param input:         :return: de + tokenizer, antonym of tokenizer.         Maps a sequence of tokens (string) in a single string.         The most simple way to do it is :python:`' '.join(tokens)`, but we         often want to remove sub-word tokenization artifacts at the same time. text to idx          :param text_a:         :param text_b:         :param max_seq_length:         :return: idx to text          :param idx:         :return: Clean up a list of simple English tokenization artifacts like         spaces before punctuations and abbreviated forms.