def tokenize_asdl(buf):
 for (lineno, line) in enumerate(buf.splitlines(), 1):
  for m in re.finditer('\\s*(\\w+|--.*|.)', line.strip()):
   c = m.group(1)
   if c[0].isalpha():
    if c[0].isupper():
     (yield Token(TokenKind.ConstructorId, c, lineno))
    else:
     (yield Token(TokenKind.TypeId, c, lineno))
   elif (c[:2] == '--'):
    break
   else:
    try:
     op_kind = TokenKind.operator_table[c]
    except KeyError:
     raise ASDLSyntaxError(('Invalid operator %s' % c), lineno)
    (yield Token(op_kind, c, lineno))

def tokenize_asdl ( buf ) : for ( lineno , line ) in enumerate ( buf . splitlines ( ) , 1 ) : for m in re . finditer ( '\\s*(\\w+|--.*|.)' , line . strip ( ) ) : c = m . group ( 1 ) if c [ 0 ] . isalpha ( ) : if c [ 0 ] . isupper ( ) : ( yield Token ( TokenKind . ConstructorId , c , lineno ) ) else : ( yield Token ( TokenKind . TypeId , c , lineno ) ) elif ( c [ : 2 ] == '--' ) : break else : try : op_kind = TokenKind . operator_table [ c ] except KeyError : raise ASDLSyntaxError ( ( 'Invalid operator %s' % c ) , lineno ) ( yield Token ( op_kind , c , lineno ) )
tokenize the given buffer .