"""Module defining tokenizers.

Tokenizers can work on string ``tf.Tensor`` as in-graph transformation.
"""

from opennmt.tokenizers.tokenizer import SpaceTokenizer, CharacterTokenizer

"""Module defining tokenizers.

Tokenizers can work on string ``tf.Tensor`` as in-graph transformation.
"""

from opennmt.tokenizers.tokenizer import SpaceTokenizer, CharacterTokenizer

Module defining tokenizers.  Tokenizers can work on string ``tf.Tensor`` as in-graph transformation.