def generate_tokens(readline, use_exact_op_types=False):
 paren_level = 0
 indents = [0]
 lnum = 0
 max = 0
 numchars = '0123456789'
 contstr = ''
 contline = None
 new_line = True
 prefix = ''
 additional_prefix = ''
 while True:
  line = readline()
  if (not line):
   if contstr:
    (yield TokenInfo(ERRORTOKEN, contstr, contstr_start, prefix))
   break
  lnum += 1
  (pos, max) = (0, len(line))
  if contstr:
   endmatch = endprog.match(line)
   if endmatch:
    pos = endmatch.end(0)
    (yield TokenInfo(STRING, (contstr + line[:pos]), contstr_start, prefix))
    contstr = ''
    contline = None
   else:
    contstr = (contstr + line)
    contline = (contline + line)
    continue
  while (pos < max):
   pseudomatch = pseudoprog.match(line, pos)
   if (not pseudomatch):
    txt = line[pos]
    if (line[pos] in '"\''):
     txt = line[pos:]
    (yield TokenInfo(ERRORTOKEN, txt, (lnum, pos), prefix))
    pos += 1
    continue
   prefix = (additional_prefix + pseudomatch.group(1))
   additional_prefix = ''
   (start, pos) = pseudomatch.span(2)
   spos = (lnum, start)
   (token, initial) = (line[start:pos], line[start])
   if (new_line and (initial not in '\r\n#')):
    new_line = False
    if (paren_level == 0):
     i = 0
     while (line[i] == '\x0c'):
      i += 1
      start -= 1
     if (start > indents[(-1)]):
      (yield TokenInfo(INDENT, '', spos, ''))
      indents.append(start)
     while (start < indents[(-1)]):
      (yield TokenInfo(DEDENT, '', spos, ''))
      indents.pop()
   if ((initial in numchars) or ((initial == '.') and (token != '.') and (token != '...'))):
    (yield TokenInfo(NUMBER, token, spos, prefix))
   elif (initial in '\r\n'):
    if ((not new_line) and (paren_level == 0)):
     (yield TokenInfo(NEWLINE, token, spos, prefix))
    else:
     additional_prefix = (prefix + token)
    new_line = True
   elif (initial == '#'):
    assert (not token.endswith('\n'))
    additional_prefix = (prefix + token)
   elif (token in triple_quoted):
    endprog = endprogs[token]
    endmatch = endprog.match(line, pos)
    if endmatch:
     pos = endmatch.end(0)
     token = line[start:pos]
     (yield TokenInfo(STRING, token, spos, prefix))
    else:
     contstr_start = (lnum, start)
     contstr = line[start:]
     contline = line
     break
   elif ((initial in single_quoted) or (token[:2] in single_quoted) or (token[:3] in single_quoted)):
    if (token[(-1)] == '\n'):
     contstr_start = (lnum, start)
     endprog = (endprogs.get(initial) or endprogs.get(token[1]) or endprogs.get(token[2]))
     contstr = line[start:]
     contline = line
     break
    else:
     (yield TokenInfo(STRING, token, spos, prefix))
   elif is_identifier(initial):
    if (token in ALWAYS_BREAK_TOKENS):
     paren_level = 0
     while True:
      indent = indents.pop()
      if (indent > start):
       (yield TokenInfo(DEDENT, '', spos, ''))
      else:
       indents.append(indent)
       break
    (yield TokenInfo(NAME, token, spos, prefix))
   elif ((initial == '\\') and (line[start:] in ('\\\n', '\\\r\n'))):
    additional_prefix += (prefix + line[start:])
    break
   else:
    if (token in '([{'):
     paren_level += 1
    elif (token in ')]}'):
     paren_level -= 1
    try:
     exact_type = opmap[token]
    except KeyError:
     exact_type = typ = ERRORTOKEN
    if use_exact_op_types:
     typ = exact_type
    else:
     typ = OP
    (yield TokenInfo(typ, token, spos, prefix))
 if (new_line or (additional_prefix[(-1):] == '\n')):
  end_pos = ((lnum + 1), 0)
 else:
  end_pos = (lnum, max)
 for indent in indents[1:]:
  (yield TokenInfo(DEDENT, '', end_pos, ''))
 (yield TokenInfo(ENDMARKER, '', end_pos, additional_prefix))

def generate_tokens ( readline , use_exact_op_types = False ) : paren_level = 0 indents = [ 0 ] lnum = 0 max = 0 numchars = '0123456789' contstr = '' contline = None new_line = True prefix = '' additional_prefix = '' while True : line = readline ( ) if ( not line ) : if contstr : ( yield TokenInfo ( ERRORTOKEN , contstr , contstr_start , prefix ) ) break lnum += 1 ( pos , max ) = ( 0 , len ( line ) ) if contstr : endmatch = endprog . match ( line ) if endmatch : pos = endmatch . end ( 0 ) ( yield TokenInfo ( STRING , ( contstr + line [ : pos ] ) , contstr_start , prefix ) ) contstr = '' contline = None else : contstr = ( contstr + line ) contline = ( contline + line ) continue while ( pos < max ) : pseudomatch = pseudoprog . match ( line , pos ) if ( not pseudomatch ) : txt = line [ pos ] if ( line [ pos ] in '"\'' ) : txt = line [ pos : ] ( yield TokenInfo ( ERRORTOKEN , txt , ( lnum , pos ) , prefix ) ) pos += 1 continue prefix = ( additional_prefix + pseudomatch . group ( 1 ) ) additional_prefix = '' ( start , pos ) = pseudomatch . span ( 2 ) spos = ( lnum , start ) ( token , initial ) = ( line [ start : pos ] , line [ start ] ) if ( new_line and ( initial not in '\r\n#' ) ) : new_line = False if ( paren_level == 0 ) : i = 0 while ( line [ i ] == '\x0c' ) : i += 1 start -= 1 if ( start > indents [ ( - 1 ) ] ) : ( yield TokenInfo ( INDENT , '' , spos , '' ) ) indents . append ( start ) while ( start < indents [ ( - 1 ) ] ) : ( yield TokenInfo ( DEDENT , '' , spos , '' ) ) indents . pop ( ) if ( ( initial in numchars ) or ( ( initial == '.' ) and ( token != '.' ) and ( token != '...' ) ) ) : ( yield TokenInfo ( NUMBER , token , spos , prefix ) ) elif ( initial in '\r\n' ) : if ( ( not new_line ) and ( paren_level == 0 ) ) : ( yield TokenInfo ( NEWLINE , token , spos , prefix ) ) else : additional_prefix = ( prefix + token ) new_line = True elif ( initial == '#' ) : assert ( not token . endswith ( '\n' ) ) additional_prefix = ( prefix + token ) elif ( token in triple_quoted ) : endprog = endprogs [ token ] endmatch = endprog . match ( line , pos ) if endmatch : pos = endmatch . end ( 0 ) token = line [ start : pos ] ( yield TokenInfo ( STRING , token , spos , prefix ) ) else : contstr_start = ( lnum , start ) contstr = line [ start : ] contline = line break elif ( ( initial in single_quoted ) or ( token [ : 2 ] in single_quoted ) or ( token [ : 3 ] in single_quoted ) ) : if ( token [ ( - 1 ) ] == '\n' ) : contstr_start = ( lnum , start ) endprog = ( endprogs . get ( initial ) or endprogs . get ( token [ 1 ] ) or endprogs . get ( token [ 2 ] ) ) contstr = line [ start : ] contline = line break else : ( yield TokenInfo ( STRING , token , spos , prefix ) ) elif is_identifier ( initial ) : if ( token in ALWAYS_BREAK_TOKENS ) : paren_level = 0 while True : indent = indents . pop ( ) if ( indent > start ) : ( yield TokenInfo ( DEDENT , '' , spos , '' ) ) else : indents . append ( indent ) break ( yield TokenInfo ( NAME , token , spos , prefix ) ) elif ( ( initial == '\\' ) and ( line [ start : ] in ( '\\\n' , '\\\r\n' ) ) ) : additional_prefix += ( prefix + line [ start : ] ) break else : if ( token in '([{' ) : paren_level += 1 elif ( token in ')]}' ) : paren_level -= 1 try : exact_type = opmap [ token ] except KeyError : exact_type = typ = ERRORTOKEN if use_exact_op_types : typ = exact_type else : typ = OP ( yield TokenInfo ( typ , token , spos , prefix ) ) if ( new_line or ( additional_prefix [ ( - 1 ) : ] == '\n' ) ) : end_pos = ( ( lnum + 1 ) , 0 ) else : end_pos = ( lnum , max ) for indent in indents [ 1 : ] : ( yield TokenInfo ( DEDENT , '' , end_pos , '' ) ) ( yield TokenInfo ( ENDMARKER , '' , end_pos , additional_prefix ) )
wrap generate_tokens to catch eof errors .