def __init__(self, alphanum_only=True, reserved_tokens=None):
    """Constructs a Tokenizer.

    Note that the Tokenizer is invertible if `alphanum_only=False`.
    i.e. `s == t.join(t.tokenize(s))`.

    Args:
      alphanum_only: `bool`, if `True`, only parse out alphanumeric tokens
        (non-alphanumeric characters are dropped);
        otherwise, keep all characters (individual tokens will still be either
        all alphanumeric or all non-alphanumeric).
      reserved_tokens: `list<str>`, a list of strings that, if any are in `s`,
        will be preserved as whole tokens, even if they contain mixed
        alphanumeric/non-alphanumeric characters.
    """
    self._alphanum_only = alphanum_only
    reserved_tokens, self._reserved_tokens_re = _prepare_reserved_tokens(
        reserved_tokens)
    self._reserved_tokens = set(reserved_tokens)
    self._alphanum_re = ALPHANUM_REGEX if self._alphanum_only else ALL_REGEX
def __init__ ( self , alphanum_only = True , reserved_tokens = None ) : self . _alphanum_only = alphanum_only reserved_tokens , self . _reserved_tokens_re = _prepare_reserved_tokens ( reserved_tokens ) self . _reserved_tokens = set ( reserved_tokens ) self . _alphanum_re = ALPHANUM_REGEX if self . _alphanum_only else ALL_REGEX
Constructs a Tokenizer.

    Note that the Tokenizer is invertible if `alphanum_only=False`.
    i.e. `s == t.join(t.tokenize(s))`.

    Args:
      alphanum_only: `bool`, if `True`, only parse out alphanumeric tokens
        (non-alphanumeric characters are dropped);
        otherwise, keep all characters (individual tokens will still be either
        all alphanumeric or all non-alphanumeric).
      reserved_tokens: `list<str>`, a list of strings that, if any are in `s`,
        will be preserved as whole tokens, even if they contain mixed
        alphanumeric/non-alphanumeric characters.