def tokenize_wrapper(input):
 skip = set((token.NEWLINE, token.INDENT, token.DEDENT))
 tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline)
 for quintuple in tokens:
  (type, value, start, end, line_text) = quintuple
  if (type not in skip):
   (yield quintuple)

def tokenize_wrapper ( input ) : skip = set ( ( token . NEWLINE , token . INDENT , token . DEDENT ) ) tokens = tokenize . generate_tokens ( StringIO . StringIO ( input ) . readline ) for quintuple in tokens : ( type , value , start , end , line_text ) = quintuple if ( type not in skip ) : ( yield quintuple )
tokenizes a string suppressing significant whitespace .