<s> """<STR_LIT>""" <EOL> __author__ = '<STR_LIT>' <EOL> __credits__ = '<STR_LIT>' <EOL> import re <EOL> from lib2to3 . pgen2 . token import * <EOL> import token <EOL> __all__ = [ x for x in dir ( token ) if x [ <NUM_LIT:0> ] != '<STR_LIT:_>' ] + [ "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" ] <EOL> del token <EOL> def group ( * choices ) : return '<STR_LIT:(>' + '<STR_LIT:|>' . join ( choices ) + '<STR_LIT:)>' <EOL> def any ( * choices ) : return group ( * choices ) + '<STR_LIT:*>' <EOL> def maybe ( * choices ) : return group ( * choices ) + '<STR_LIT:?>' <EOL> Whitespace = r'<STR_LIT>' <EOL> Comment = r'<STR_LIT>' <EOL> Ignore = Whitespace + any ( r'<STR_LIT>' + Whitespace ) + maybe ( Comment ) <EOL> Name = r'<STR_LIT>' <EOL> Binnumber = r'<STR_LIT>' <EOL> Hexnumber = r'<STR_LIT>' <EOL> Octnumber = r'<STR_LIT>' <EOL> Decnumber = r'<STR_LIT>' <EOL> Intnumber = group ( Binnumber , Hexnumber , Octnumber , Decnumber ) <EOL> Exponent = r'<STR_LIT>' <EOL> Pointfloat = group ( r'<STR_LIT>' , r'<STR_LIT>' ) + maybe ( Exponent ) <EOL> Expfloat = r'<STR_LIT>' + Exponent <EOL> Floatnumber = group ( Pointfloat , Expfloat ) <EOL> Imagnumber = group ( r'<STR_LIT>' , Floatnumber + r'<STR_LIT>' ) <EOL> Number = group ( Imagnumber , Floatnumber , Intnumber ) <EOL> Single = r"<STR_LIT>" <EOL> Double = r'<STR_LIT>' <EOL> Single3 = r"<STR_LIT>" <EOL> Double3 = r'<STR_LIT>' <EOL> Triple = group ( "<STR_LIT>" , '<STR_LIT>' ) <EOL> String = group ( r"<STR_LIT>" , <EOL> r'<STR_LIT>' ) <EOL> Operator = group ( r"<STR_LIT>" , r"<STR_LIT>" , r"<STR_LIT>" , r"<STR_LIT>" , r"<STR_LIT>" , <EOL> r"<STR_LIT>" , r"<STR_LIT>" , <EOL> r"<STR_LIT>" , <EOL> r"<STR_LIT>" ) <EOL> Bracket = '<STR_LIT>' <EOL> Special = group ( r'<STR_LIT>' , r'<STR_LIT>' ) <EOL> Funny = group ( Operator , Bracket , Special ) <EOL> PlainToken = group ( Number , Funny , String , Name ) <EOL> Token = Ignore + PlainToken <EOL> ContStr = group ( r"<STR_LIT>" + <EOL> group ( "<STR_LIT:'>" , r'<STR_LIT>' ) , <EOL> r'<STR_LIT>' + <EOL> group ( '<STR_LIT:">' , r'<STR_LIT>' ) ) <EOL> PseudoExtras = group ( r'<STR_LIT>' , Comment , Triple ) <EOL> PseudoToken = Whitespace + group ( PseudoExtras , Number , Funny , ContStr , Name ) <EOL> tokenprog , pseudoprog , single3prog , double3prog = list ( map ( <EOL> re . compile , ( Token , PseudoToken , Single3 , Double3 ) ) ) <EOL> endprogs = { "<STR_LIT:'>" : re . compile ( Single ) , '<STR_LIT:">' : re . compile ( Double ) , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> "<STR_LIT>" : single3prog , '<STR_LIT>' : double3prog , <EOL> '<STR_LIT:r>' : None , '<STR_LIT:R>' : None , <EOL> '<STR_LIT:u>' : None , '<STR_LIT>' : None , <EOL> '<STR_LIT:b>' : None , '<STR_LIT:B>' : None } <EOL> triple_quoted = { } <EOL> for t in ( "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , ) : <EOL> triple_quoted [ t ] = t <EOL> single_quoted = { } <EOL> for t in ( "<STR_LIT:'>" , '<STR_LIT:">' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , <EOL> "<STR_LIT>" , '<STR_LIT>' , "<STR_LIT>" , '<STR_LIT>' , ) : <EOL> single_quoted [ t ] = t <EOL> tabsize = <NUM_LIT:8> <EOL> class TokenError ( Exception ) : pass <EOL> class StopTokenizing ( Exception ) : pass <EOL> def printtoken ( type , token , xxx_todo_changeme , xxx_todo_changeme1 , line ) : <EOL> ( srow , scol ) = xxx_todo_changeme <EOL> ( erow , ecol ) = xxx_todo_changeme1 <EOL> print ( "<STR_LIT>" % ( srow , scol , erow , ecol , tok_name [ type ] , repr ( token ) ) ) <EOL> def tokenize ( readline , tokeneater = printtoken ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> tokenize_loop ( readline , tokeneater ) <EOL> except StopTokenizing : <EOL> pass <EOL> def tokenize_loop ( readline , tokeneater ) : <EOL> for token_info in generate_tokens ( readline ) : <EOL> tokeneater ( * token_info ) <EOL> class Untokenizer : <EOL> def __init__ ( self ) : <EOL> self . tokens = [ ] <EOL> self . prev_row = <NUM_LIT:1> <EOL> self . prev_col = <NUM_LIT:0> <EOL> def add_whitespace ( self , start ) : <EOL> row , col = start <EOL> assert row <= self . prev_row <EOL> col_offset = col - self . prev_col <EOL> if col_offset : <EOL> self . tokens . append ( "<STR_LIT:U+0020>" * col_offset ) <EOL> def untokenize ( self , iterable ) : <EOL> for t in iterable : <EOL> if len ( t ) == <NUM_LIT:2> : <EOL> self . compat ( t , iterable ) <EOL> break <EOL> tok_type , token , start , end , line = t <EOL> self . add_whitespace ( start ) <EOL> self . tokens . append ( token ) <EOL> self . prev_row , self . prev_col = end <EOL> if tok_type in ( NEWLINE , NL ) : <EOL> self . prev_row += <NUM_LIT:1> <EOL> self . prev_col = <NUM_LIT:0> <EOL> return "<STR_LIT>" . join ( self . tokens ) <EOL> def compat ( self , token , iterable ) : <EOL> startline = False <EOL> indents = [ ] <EOL> toks_append = self . tokens . append <EOL> toknum , tokval = token <EOL> if toknum in ( NAME , NUMBER ) : <EOL> tokval += '<STR_LIT:U+0020>' <EOL> if toknum in ( NEWLINE , NL ) : <EOL> startline = True <EOL> for tok in iterable : <EOL> toknum , tokval = tok [ : <NUM_LIT:2> ] <EOL> if toknum in ( NAME , NUMBER ) : <EOL> tokval += '<STR_LIT:U+0020>' <EOL> if toknum == INDENT : <EOL> indents . append ( tokval ) <EOL> continue <EOL> elif toknum == DEDENT : <EOL> indents . pop ( ) <EOL> continue <EOL> elif toknum in ( NEWLINE , NL ) : <EOL> startline = True <EOL> elif startline and indents : <EOL> toks_append ( indents [ - <NUM_LIT:1> ] ) <EOL> startline = False <EOL> toks_append ( tokval ) <EOL> cookie_re = re . compile ( "<STR_LIT>" ) <EOL> def detect_encoding ( readline ) : <EOL> """<STR_LIT>""" <EOL> from codecs import BOM_UTF8 , lookup <EOL> bom_found = False <EOL> encoding = None <EOL> def read_or_stop ( ) : <EOL> try : <EOL> return readline ( ) <EOL> except StopIteration : <EOL> return b <EOL> def find_cookie ( line ) : <EOL> try : <EOL> line_string = line . decode ( '<STR_LIT:ascii>' ) <EOL> except UnicodeDecodeError : <EOL> return None <EOL> matches = cookie_re . findall ( line_string ) <EOL> if not matches : <EOL> return None <EOL> encoding = matches [ <NUM_LIT:0> ] <EOL> try : <EOL> codec = lookup ( encoding ) <EOL> except LookupError : <EOL> raise SyntaxError ( "<STR_LIT>" + encoding ) <EOL> if bom_found and codec . name != '<STR_LIT:utf-8>' : <EOL> raise SyntaxError ( '<STR_LIT>' ) <EOL> return encoding <EOL> first = read_or_stop ( ) <EOL> if first . startswith ( BOM_UTF8 ) : <EOL> bom_found = True <EOL> first = first [ <NUM_LIT:3> : ] <EOL> if not first : <EOL> return '<STR_LIT:utf-8>' , [ ] <EOL> encoding = find_cookie ( first ) <EOL> if encoding : <EOL> return encoding , [ first ] <EOL> second = read_or_stop ( ) <EOL> if not second : <EOL> return '<STR_LIT:utf-8>' , [ first ] <EOL> encoding = find_cookie ( second ) <EOL> if encoding : <EOL> return encoding , [ first , second ] <EOL> return '<STR_LIT:utf-8>' , [ first , second ] <EOL> def untokenize ( iterable ) : <EOL> """<STR_LIT>""" <EOL> ut = Untokenizer ( ) <EOL> return ut . untokenize ( iterable ) <EOL> def generate_tokens ( readline ) : <EOL> """<STR_LIT>""" <EOL> lnum = parenlev = continued = <NUM_LIT:0> <EOL> namechars = '<STR_LIT>' + '<STR_LIT:_>' <EOL> numchars = '<STR_LIT>' <EOL> contstr , needcont = '<STR_LIT>' , <NUM_LIT:0> <EOL> contline = None <EOL> indents = [ <NUM_LIT:0> ] <EOL> while <NUM_LIT:1> : <EOL> try : <EOL> line = readline ( ) <EOL> except StopIteration : <EOL> line = '<STR_LIT>' <EOL> lnum = lnum + <NUM_LIT:1> <EOL> pos , max = <NUM_LIT:0> , len ( line ) <EOL> if contstr : <EOL> if not line : <EOL> raise TokenError ( "<STR_LIT>" , strstart ) <EOL> endmatch = endprog . match ( line ) <EOL> if endmatch : <EOL> pos = end = endmatch . end ( <NUM_LIT:0> ) <EOL> yield ( STRING , contstr + line [ : end ] , <EOL> strstart , ( lnum , end ) , contline + line ) <EOL> contstr , needcont = '<STR_LIT>' , <NUM_LIT:0> <EOL> contline = None <EOL> elif needcont and line [ - <NUM_LIT:2> : ] != '<STR_LIT>' and line [ - <NUM_LIT:3> : ] != '<STR_LIT>' : <EOL> yield ( ERRORTOKEN , contstr + line , <EOL> strstart , ( lnum , len ( line ) ) , contline ) <EOL> contstr = '<STR_LIT>' <EOL> contline = None <EOL> continue <EOL> else : <EOL> contstr = contstr + line <EOL> contline = contline + line <EOL> continue <EOL> elif parenlev == <NUM_LIT:0> and not continued : <EOL> if not line : break <EOL> column = <NUM_LIT:0> <EOL> while pos < max : <EOL> if line [ pos ] == '<STR_LIT:U+0020>' : column = column + <NUM_LIT:1> <EOL> elif line [ pos ] == '<STR_LIT:\t>' : column = ( column / tabsize + <NUM_LIT:1> ) * tabsize <EOL> elif line [ pos ] == '<STR_LIT>' : column = <NUM_LIT:0> <EOL> else : break <EOL> pos = pos + <NUM_LIT:1> <EOL> if pos == max : break <EOL> if line [ pos ] in '<STR_LIT>' : <EOL> if line [ pos ] == '<STR_LIT:#>' : <EOL> comment_token = line [ pos : ] . rstrip ( '<STR_LIT:\r\n>' ) <EOL> nl_pos = pos + len ( comment_token ) <EOL> yield ( COMMENT , comment_token , <EOL> ( lnum , pos ) , ( lnum , pos + len ( comment_token ) ) , line ) <EOL> yield ( NL , line [ nl_pos : ] , <EOL> ( lnum , nl_pos ) , ( lnum , len ( line ) ) , line ) <EOL> else : <EOL> yield ( ( NL , COMMENT ) [ line [ pos ] == '<STR_LIT:#>' ] , line [ pos : ] , <EOL> ( lnum , pos ) , ( lnum , len ( line ) ) , line ) <EOL> continue <EOL> if column > indents [ - <NUM_LIT:1> ] : <EOL> indents . append ( column ) <EOL> yield ( INDENT , line [ : pos ] , ( lnum , <NUM_LIT:0> ) , ( lnum , pos ) , line ) <EOL> while column < indents [ - <NUM_LIT:1> ] : <EOL> if column not in indents : <EOL> raise IndentationError ( <EOL> "<STR_LIT>" , <EOL> ( "<STR_LIT>" , lnum , pos , line ) ) <EOL> indents = indents [ : - <NUM_LIT:1> ] <EOL> yield ( DEDENT , '<STR_LIT>' , ( lnum , pos ) , ( lnum , pos ) , line ) <EOL> else : <EOL> if not line : <EOL> raise TokenError ( "<STR_LIT>" , ( lnum , <NUM_LIT:0> ) ) <EOL> continued = <NUM_LIT:0> <EOL> while pos < max : <EOL> pseudomatch = pseudoprog . match ( line , pos ) <EOL> if pseudomatch : <EOL> start , end = pseudomatch . span ( <NUM_LIT:1> ) <EOL> spos , epos , pos = ( lnum , start ) , ( lnum , end ) , end <EOL> token , initial = line [ start : end ] , line [ start ] <EOL> if initial in numchars or ( initial == '<STR_LIT:.>' and token != '<STR_LIT:.>' ) : <EOL> yield ( NUMBER , token , spos , epos , line ) <EOL> elif initial in '<STR_LIT:\r\n>' : <EOL> newline = NEWLINE <EOL> if parenlev > <NUM_LIT:0> : <EOL> newline = NL <EOL> yield ( newline , token , spos , epos , line ) <EOL> elif initial == '<STR_LIT:#>' : <EOL> assert not token . endswith ( "<STR_LIT:\n>" ) <EOL> yield ( COMMENT , token , spos , epos , line ) <EOL> elif token in triple_quoted : <EOL> endprog = endprogs [ token ] <EOL> endmatch = endprog . match ( line , pos ) <EOL> if endmatch : <EOL> pos = endmatch . end ( <NUM_LIT:0> ) <EOL> token = line [ start : pos ] <EOL> yield ( STRING , token , spos , ( lnum , pos ) , line ) <EOL> else : <EOL> strstart = ( lnum , start ) <EOL> contstr = line [ start : ] <EOL> contline = line <EOL> break <EOL> elif initial in single_quoted or token [ : <NUM_LIT:2> ] in single_quoted or token [ : <NUM_LIT:3> ] in single_quoted : <EOL> if token [ - <NUM_LIT:1> ] == '<STR_LIT:\n>' : <EOL> strstart = ( lnum , start ) <EOL> endprog = ( endprogs [ initial ] or endprogs [ token [ <NUM_LIT:1> ] ] or <EOL> endprogs [ token [ <NUM_LIT:2> ] ] ) <EOL> contstr , needcont = line [ start : ] , <NUM_LIT:1> <EOL> contline = line <EOL> break <EOL> else : <EOL> yield ( STRING , token , spos , epos , line ) <EOL> elif initial in namechars : <EOL> yield ( NAME , token , spos , epos , line ) <EOL> elif initial == '<STR_LIT:\\>' : <EOL> yield ( NL , token , spos , ( lnum , pos ) , line ) <EOL> continued = <NUM_LIT:1> <EOL> else : <EOL> if initial in '<STR_LIT>' : parenlev = parenlev + <NUM_LIT:1> <EOL> elif initial in '<STR_LIT>' : parenlev = parenlev - <NUM_LIT:1> <EOL> yield ( OP , token , spos , epos , line ) <EOL> else : <EOL> yield ( ERRORTOKEN , line [ pos ] , <EOL> ( lnum , pos ) , ( lnum , pos + <NUM_LIT:1> ) , line ) <EOL> pos = pos + <NUM_LIT:1> <EOL> for indent in indents [ <NUM_LIT:1> : ] : <EOL> yield ( DEDENT , '<STR_LIT>' , ( lnum , <NUM_LIT:0> ) , ( lnum , <NUM_LIT:0> ) , '<STR_LIT>' ) <EOL> yield ( ENDMARKER , '<STR_LIT>' , ( lnum , <NUM_LIT:0> ) , ( lnum , <NUM_LIT:0> ) , '<STR_LIT>' ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> import sys <EOL> if len ( sys . argv ) > <NUM_LIT:1> : tokenize ( open ( sys . argv [ <NUM_LIT:1> ] ) . readline ) <EOL> else : tokenize ( sys . stdin . readline ) </s>
