<s> """<STR_LIT>""" <EOL> from __future__ import print_function <EOL> import sys , re <EOL> MIN_WINDOW = <NUM_LIT> <EOL> class SyntaxError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , pos = None , msg = "<STR_LIT>" , context = None ) : <EOL> Exception . __init__ ( self ) <EOL> self . pos = pos <EOL> self . msg = msg <EOL> self . context = context <EOL> def __str__ ( self ) : <EOL> if not self . pos : return '<STR_LIT>' <EOL> else : return '<STR_LIT>' % ( repr ( self . pos ) , self . msg ) <EOL> class NoMoreTokens ( Exception ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class Token ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , type , value , pos = None ) : <EOL> """<STR_LIT>""" <EOL> self . type = type <EOL> self . value = value <EOL> self . pos = pos <EOL> def __repr__ ( self ) : <EOL> output = '<STR_LIT>' % ( self . type , repr ( self . value ) ) <EOL> if self . pos : <EOL> output += "<STR_LIT>" <EOL> if self . pos [ <NUM_LIT:0> ] : <EOL> output += "<STR_LIT>" % self . pos [ <NUM_LIT:0> ] <EOL> if self . pos [ <NUM_LIT:1> ] : <EOL> output += "<STR_LIT>" % self . pos [ <NUM_LIT:1> ] <EOL> if self . pos [ <NUM_LIT:2> ] is not None : <EOL> output += "<STR_LIT>" % self . pos [ <NUM_LIT:2> ] <EOL> output += "<STR_LIT:>>" <EOL> return output <EOL> in_name = <NUM_LIT:0> <EOL> class Scanner ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , patterns , ignore , input = "<STR_LIT>" , <EOL> file = None , filename = None , stacked = False ) : <EOL> """<STR_LIT>""" <EOL> if not filename : <EOL> global in_name <EOL> filename = "<STR_LIT>" % in_name <EOL> in_name += <NUM_LIT:1> <EOL> self . input = input <EOL> self . ignore = ignore <EOL> self . file = file <EOL> self . filename = filename <EOL> self . pos = <NUM_LIT:0> <EOL> self . del_pos = <NUM_LIT:0> <EOL> self . line = <NUM_LIT:1> <EOL> self . del_line = <NUM_LIT:0> <EOL> self . col = <NUM_LIT:0> <EOL> self . tokens = [ ] <EOL> self . stack = None <EOL> self . stacked = stacked <EOL> self . last_read_token = None <EOL> self . last_token = None <EOL> self . last_types = None <EOL> if patterns is not None : <EOL> self . patterns = [ ] <EOL> for terminal , regex in patterns : <EOL> self . patterns . append ( ( terminal , re . compile ( regex ) ) ) <EOL> def stack_input ( self , input = "<STR_LIT>" , file = None , filename = None ) : <EOL> """<STR_LIT>""" <EOL> if self . stack : <EOL> if not filename : <EOL> filename = <NUM_LIT:1> <EOL> else : <EOL> try : <EOL> filename += <NUM_LIT:1> <EOL> except TypeError : <EOL> pass <EOL> self . stack . stack_input ( input , file , filename ) <EOL> else : <EOL> try : <EOL> filename += <NUM_LIT:0> <EOL> except TypeError : <EOL> pass <EOL> else : <EOL> filename = "<STR_LIT>" % filename <EOL> self . stack = self . __class__ ( input , file , filename , stacked = True ) <EOL> def get_pos ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . stack : return self . stack . get_pos ( ) <EOL> return ( self . filename , self . line + self . del_line , self . col ) <EOL> def print_line_with_pointer ( self , pos , length = <NUM_LIT:0> , out = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> file , line , p = pos <EOL> if file != self . filename : <EOL> if self . stack : return self . stack . print_line_with_pointer ( pos , length = length , out = out ) <EOL> print >> out , "<STR_LIT>" % file <EOL> return <EOL> text = self . input <EOL> p += length - <NUM_LIT:1> <EOL> origline = line <EOL> line -= self . del_line <EOL> spos = <NUM_LIT:0> <EOL> if line > <NUM_LIT:0> : <EOL> while <NUM_LIT:1> : <EOL> line = line - <NUM_LIT:1> <EOL> try : <EOL> cr = text . index ( "<STR_LIT:\n>" , spos ) <EOL> except ValueError : <EOL> if line : <EOL> text = "<STR_LIT>" <EOL> break <EOL> if line == <NUM_LIT:0> : <EOL> text = text [ spos : cr ] <EOL> break <EOL> spos = cr + <NUM_LIT:1> <EOL> else : <EOL> print >> out , "<STR_LIT>" % ( file , origline ) <EOL> return <EOL> text = text [ max ( p - <NUM_LIT> , <NUM_LIT:0> ) : p + <NUM_LIT> ] <EOL> p = p - max ( p - <NUM_LIT> , <NUM_LIT:0> ) <EOL> i = text [ : p ] . rfind ( '<STR_LIT:\n>' ) <EOL> j = text [ : p ] . rfind ( '<STR_LIT:\r>' ) <EOL> if i < <NUM_LIT:0> or ( <NUM_LIT:0> <= j < i ) : i = j <EOL> if <NUM_LIT:0> <= i < p : <EOL> p = p - i - <NUM_LIT:1> <EOL> text = text [ i + <NUM_LIT:1> : ] <EOL> i = text . find ( '<STR_LIT:\n>' , p ) <EOL> j = text . find ( '<STR_LIT:\r>' , p ) <EOL> if i < <NUM_LIT:0> or ( <NUM_LIT:0> <= j < i ) : i = j <EOL> if i >= <NUM_LIT:0> : <EOL> text = text [ : i ] <EOL> while len ( text ) > <NUM_LIT> and p > <NUM_LIT> : <EOL> text = "<STR_LIT>" + text [ <NUM_LIT:10> : ] <EOL> p = p - <NUM_LIT:7> <EOL> print >> out , '<STR_LIT>' , text <EOL> print >> out , '<STR_LIT>' , '<STR_LIT:U+0020>' * p + '<STR_LIT>' <EOL> def grab_input ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . file : return <EOL> if len ( self . input ) - self . pos >= MIN_WINDOW : return <EOL> data = self . file . read ( MIN_WINDOW ) <EOL> if data is None or data == "<STR_LIT>" : <EOL> self . file = None <EOL> if self . pos > <NUM_LIT:2> * MIN_WINDOW : <EOL> self . del_pos += MIN_WINDOW <EOL> self . del_line += self . input [ : MIN_WINDOW ] . count ( "<STR_LIT:\n>" ) <EOL> self . pos -= MIN_WINDOW <EOL> self . input = self . input [ MIN_WINDOW : ] + data <EOL> else : <EOL> self . input = self . input + data <EOL> def getchar ( self ) : <EOL> """<STR_LIT>""" <EOL> self . grab_input ( ) <EOL> c = self . input [ self . pos ] <EOL> self . pos += <NUM_LIT:1> <EOL> return c <EOL> def token ( self , restrict , context = None ) : <EOL> """<STR_LIT>""" <EOL> while <NUM_LIT:1> : <EOL> if self . stack : <EOL> try : <EOL> return self . stack . token ( restrict , context ) <EOL> except StopIteration : <EOL> self . stack = None <EOL> self . grab_input ( ) <EOL> if self . stacked and self . pos == len ( self . input ) : <EOL> raise StopIteration <EOL> best_match = - <NUM_LIT:1> <EOL> best_pat = '<STR_LIT>' <EOL> best_m = None <EOL> for p , regexp in self . patterns : <EOL> if restrict and p not in restrict and p not in self . ignore : <EOL> continue <EOL> m = regexp . match ( self . input , self . pos ) <EOL> if m and m . end ( ) - m . start ( ) > best_match : <EOL> best_pat = p <EOL> best_match = m . end ( ) - m . start ( ) <EOL> best_m = m <EOL> if best_pat == '<STR_LIT>' and best_match < <NUM_LIT:0> : <EOL> msg = '<STR_LIT>' <EOL> if restrict : <EOL> msg = '<STR_LIT>' + '<STR_LIT:U+002CU+0020>' . join ( restrict ) <EOL> raise SyntaxError ( self . get_pos ( ) , msg , context = context ) <EOL> ignore = best_pat in self . ignore <EOL> value = self . input [ self . pos : self . pos + best_match ] <EOL> if not ignore : <EOL> tok = Token ( type = best_pat , value = value , pos = self . get_pos ( ) ) <EOL> self . pos += best_match <EOL> npos = value . rfind ( "<STR_LIT:\n>" ) <EOL> if npos > - <NUM_LIT:1> : <EOL> self . col = best_match - npos <EOL> self . line += value . count ( "<STR_LIT:\n>" ) <EOL> else : <EOL> self . col += best_match <EOL> if not ignore : <EOL> if len ( self . tokens ) >= <NUM_LIT:10> : <EOL> del self . tokens [ <NUM_LIT:0> ] <EOL> self . tokens . append ( tok ) <EOL> self . last_read_token = tok <EOL> return tok <EOL> else : <EOL> ignore = self . ignore [ best_pat ] <EOL> if ignore : <EOL> ignore ( self , best_m ) <EOL> def peek ( self , * types , ** kw ) : <EOL> """<STR_LIT>""" <EOL> context = kw . get ( "<STR_LIT>" , None ) <EOL> if self . last_token is None : <EOL> self . last_types = types <EOL> self . last_token = self . token ( types , context ) <EOL> elif self . last_types : <EOL> for t in types : <EOL> if t not in self . last_types : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> return self . last_token . type <EOL> def scan ( self , type , ** kw ) : <EOL> """<STR_LIT>""" <EOL> context = kw . get ( "<STR_LIT>" , None ) <EOL> if self . last_token is None : <EOL> tok = self . token ( [ type ] , context ) <EOL> else : <EOL> if self . last_types and type not in self . last_types : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> tok = self . last_token <EOL> self . last_token = None <EOL> if tok . type != type : <EOL> if not self . last_types : self . last_types = [ ] <EOL> raise SyntaxError ( tok . pos , '<STR_LIT>' + type + '<STR_LIT>' + '<STR_LIT:U+002CU+0020>' . join ( self . last_types ) + "<STR_LIT>" + tok . type , context = context ) <EOL> return tok . value <EOL> class Parser ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , scanner ) : <EOL> self . _scanner = scanner <EOL> def _stack ( self , input = "<STR_LIT>" , file = None , filename = None ) : <EOL> """<STR_LIT>""" <EOL> self . _scanner . stack_input ( input , file , filename ) <EOL> self . _tok = None <EOL> def _peek ( self , * types , ** kw ) : <EOL> """<STR_LIT>""" <EOL> return self . _scanner . peek ( * types , ** kw ) <EOL> def _scan ( self , type , ** kw ) : <EOL> """<STR_LIT>""" <EOL> return self . _scanner . scan ( type , ** kw ) <EOL> class Context ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , parent , scanner , rule , args = ( ) ) : <EOL> """<STR_LIT>""" <EOL> self . parent = parent <EOL> self . scanner = scanner <EOL> self . rule = rule <EOL> self . args = args <EOL> while scanner . stack : scanner = scanner . stack <EOL> self . token = scanner . last_read_token <EOL> def __str__ ( self ) : <EOL> output = '<STR_LIT>' <EOL> if self . parent : output = str ( self . parent ) + '<STR_LIT>' <EOL> output += self . rule <EOL> return output <EOL> def print_error ( err , scanner , max_ctx = None ) : <EOL> """<STR_LIT>""" <EOL> pos = err . pos <EOL> if not pos : <EOL> pos = scanner . get_pos ( ) <EOL> file_name , line_number , column_number = pos <EOL> print ( '<STR_LIT>' % ( file_name , line_number , column_number , err . msg ) , file = sys . stderr ) <EOL> scanner . print_line_with_pointer ( pos ) <EOL> context = err . context <EOL> token = None <EOL> while context : <EOL> print ( '<STR_LIT>' % ( context . rule , tuple ( context . args ) ) , file = sys . stderr ) <EOL> if context . token : <EOL> token = context . token <EOL> if token : <EOL> scanner . print_line_with_pointer ( token . pos , length = len ( token . value ) ) <EOL> context = context . parent <EOL> if max_ctx : <EOL> max_ctx = max_ctx - <NUM_LIT:1> <EOL> if not max_ctx : <EOL> break <EOL> def wrap_error_reporter ( parser , rule , * args , ** kw ) : <EOL> try : <EOL> return getattr ( parser , rule ) ( * args , ** kw ) <EOL> except SyntaxError as e : <EOL> print_error ( e , parser . _scanner ) <EOL> except NoMoreTokens : <EOL> print ( '<STR_LIT>' , file = sys . stderr ) <EOL> print ( parser . _scanner , file = sys . stderr ) </s>
