<s> '''<STR_LIT>''' <EOL> import re <EOL> from . tokens import Token <EOL> EOF = '<STR_LIT>' <EOL> class ScannerState ( object ) : <EOL> def __init__ ( self , source ) : <EOL> self . source = source <EOL> self . position = <NUM_LIT:0> <EOL> self . start = <NUM_LIT:0> <EOL> def consume ( self ) : <EOL> '''<STR_LIT>''' <EOL> if self . position >= len ( self . source ) : <EOL> return EOF <EOL> rv = self . source [ self . position ] <EOL> self . position += <NUM_LIT:1> <EOL> return rv <EOL> def backup ( self ) : <EOL> '''<STR_LIT>''' <EOL> self . position -= <NUM_LIT:1> <EOL> def ignore ( self ) : <EOL> '''<STR_LIT>''' <EOL> self . start = self . position <EOL> def emit ( self ) : <EOL> '''<STR_LIT>''' <EOL> content = self . source [ self . start : self . position ] <EOL> self . ignore ( ) <EOL> return content <EOL> def skip ( self , character ) : <EOL> '''<STR_LIT>''' <EOL> while True : <EOL> c = self . consume ( ) <EOL> if c == EOF or c != character : <EOL> break <EOL> if c != EOF : <EOL> self . backup ( ) <EOL> def skip_run ( self , characters ) : <EOL> '''<STR_LIT>''' <EOL> while True : <EOL> c = self . consume ( ) <EOL> if c == EOF or c not in characters : <EOL> break <EOL> if c != EOF : <EOL> self . backup ( ) <EOL> def expect ( self , item , on_error = None ) : <EOL> '''<STR_LIT>''' <EOL> c = self . consume ( ) <EOL> if c != item : <EOL> if on_error : <EOL> raise on_error ( ) <EOL> raise ValueError ( '<STR_LIT>' . format ( item , c ) ) <EOL> return c <EOL> def expect_match ( self , pattern , on_error = None ) : <EOL> '''<STR_LIT>''' <EOL> m = re . compile ( pattern ) . match ( self . source , self . position ) <EOL> if m : <EOL> self . position += m . end ( ) - m . start ( ) <EOL> return m <EOL> if not on_error : <EOL> raise ValueError ( '<STR_LIT>' . format ( pattern , self . source [ self . position : ] ) ) <EOL> raise on_error ( ) <EOL> def peek ( self , item ) : <EOL> '''<STR_LIT>''' <EOL> return self . source [ self . position : self . position + len ( item ) ] == item <EOL> def match ( self , pattern ) : <EOL> '''<STR_LIT>''' <EOL> m = re . compile ( pattern ) . match ( self . source , self . position ) <EOL> if m : <EOL> self . position += m . end ( ) - m . start ( ) <EOL> return m <EOL> return </s>
