def tokenize_string(source):
 line_reader = StringIO(source).readline
 for (toknum, tokval, _, _, _) in tokenize.generate_tokens(line_reader):
  (yield (toknum, tokval))

def tokenize_string ( source ) : line_reader = StringIO ( source ) . readline for ( toknum , tokval , _ , _ , _ ) in tokenize . generate_tokens ( line_reader ) : ( yield ( toknum , tokval ) )
tokenize a python source code string .