def number_aware_tokenizer(doc):
 token_pattern = re.compile(u'(?u)\\b\\w\\w+\\b')
 tokens = token_pattern.findall(doc)
 tokens = [('#NUMBER' if (token[0] in '0123456789_') else token) for token in tokens]
 return tokens

def number_aware_tokenizer ( doc ) : token_pattern = re . compile ( u'(?u)\\b\\w\\w+\\b' ) tokens = token_pattern . findall ( doc ) tokens = [ ( '#NUMBER' if ( token [ 0 ] in '0123456789_' ) else token ) for token in tokens ] return tokens
tokenizer that maps all numeric tokens to a placeholder .