<reponame>asyml/asyml-utilities<filename>asyml_utilities/special_tokens.py
__all__ = [
    'SpecialTokens',
]


class SpecialTokens:
    r"""Special tokens, including :attr:`PAD`, :attr:`BOS`, :attr:`EOS`,
    :attr:`UNK`. These tokens will by default have token ids 0, 1, 2, 3,
    respectively.
    """
    PAD = "<PAD>"
    BOS = "<BOS>"
    EOS = "<EOS>"
    UNK = "<UNK>"

__all__ = [
    'SpecialTokens',
]


class SpecialTokens:
    r"""Special tokens, including :attr:`PAD`, :attr:`BOS`, :attr:`EOS`,
    :attr:`UNK`. These tokens will by default have token ids 0, 1, 2, 3,
    respectively.
    """
    PAD = "<PAD>"
    BOS = "<BOS>"
    EOS = "<EOS>"
    UNK = "<UNK>"
Special tokens, including :attr:`PAD`, :attr:`BOS`, :attr:`EOS`,     :attr:`UNK`. These tokens will by default have token ids 0, 1, 2, 3,     respectively.