def getTokens(command):
 if (type(command) == unicode):
  try:
   command = command.encode(wx.GetDefaultPyEncoding())
  except UnicodeEncodeError:
   pass
 f = cStringIO.StringIO(command)
 tokens = []
 try:
  def eater(*args):
   tokens.append(args)
  tokenize.tokenize_loop(f.readline, eater)
 except tokenize.TokenError:
  pass
 return tokens

def getTokens ( command ) : if ( type ( command ) == unicode ) : try : command = command . encode ( wx . GetDefaultPyEncoding ( ) ) except UnicodeEncodeError : pass f = cStringIO . StringIO ( command ) tokens = [ ] try : def eater ( * args ) : tokens . append ( args ) tokenize . tokenize_loop ( f . readline , eater ) except tokenize . TokenError : pass return tokens
return list of token tuples for command .