def basic_tokenizer(sentence, _WORD_SPLIT=re.compile('([.,!?"\':;)(])')):
 words = []
 sentence = tf.compat.as_bytes(sentence)
 for space_separated_fragment in sentence.strip().split():
  words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
 return [w for w in words if w]

def basic_tokenizer ( sentence , _WORD_SPLIT = re . compile ( '([.,!?"\':;)(])' ) ) : words = [ ] sentence = tf . compat . as_bytes ( sentence ) for space_separated_fragment in sentence . strip ( ) . split ( ) : words . extend ( re . split ( _WORD_SPLIT , space_separated_fragment ) ) return [ w for w in words if w ]
very basic tokenizer: split the sentence into a list of tokens .