def sentence_to_token_ids(sentence, vocabulary, tokenizer=None, normalize_digits=True, UNK_ID=3, _DIGIT_RE=re.compile('\\d')):
 if tokenizer:
  words = tokenizer(sentence)
 else:
  words = basic_tokenizer(sentence)
 if (not normalize_digits):
  return [vocabulary.get(w, UNK_ID) for w in words]
 return [vocabulary.get(re.sub(_DIGIT_RE, '0', w), UNK_ID) for w in words]

def sentence_to_token_ids ( sentence , vocabulary , tokenizer = None , normalize_digits = True , UNK_ID = 3 , _DIGIT_RE = re . compile ( '\\d' ) ) : if tokenizer : words = tokenizer ( sentence ) else : words = basic_tokenizer ( sentence ) if ( not normalize_digits ) : return [ vocabulary . get ( w , UNK_ID ) for w in words ] return [ vocabulary . get ( re . sub ( _DIGIT_RE , '0' , w ) , UNK_ID ) for w in words ]
convert a string to list of integers representing token-ids .