<s> import Errors <EOL> from Regexps import BOL , EOL , EOF <EOL> class Scanner : <EOL> """<STR_LIT>""" <EOL> lexicon = None <EOL> stream = None <EOL> name = '<STR_LIT>' <EOL> buffer = '<STR_LIT>' <EOL> buf_start_pos = <NUM_LIT:0> <EOL> next_pos = <NUM_LIT:0> <EOL> cur_pos = <NUM_LIT:0> <EOL> cur_line = <NUM_LIT:1> <EOL> cur_line_start = <NUM_LIT:0> <EOL> start_pos = <NUM_LIT:0> <EOL> start_line = <NUM_LIT:0> <EOL> start_col = <NUM_LIT:0> <EOL> text = None <EOL> initial_state = None <EOL> state_name = '<STR_LIT>' <EOL> queue = None <EOL> trace = <NUM_LIT:0> <EOL> def __init__ ( self , lexicon , stream , name = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> self . lexicon = lexicon <EOL> self . stream = stream <EOL> self . name = name <EOL> self . queue = [ ] <EOL> self . initial_state = None <EOL> self . begin ( '<STR_LIT>' ) <EOL> self . next_pos = <NUM_LIT:0> <EOL> self . cur_pos = <NUM_LIT:0> <EOL> self . cur_line_start = <NUM_LIT:0> <EOL> self . cur_char = BOL <EOL> self . input_state = <NUM_LIT:1> <EOL> def read ( self ) : <EOL> """<STR_LIT>""" <EOL> queue = self . queue <EOL> while not queue : <EOL> self . text , action = self . scan_a_token ( ) <EOL> if action is None : <EOL> self . produce ( None ) <EOL> self . eof ( ) <EOL> else : <EOL> value = action . perform ( self , self . text ) <EOL> if value is not None : <EOL> self . produce ( value ) <EOL> result = queue [ <NUM_LIT:0> ] <EOL> del queue [ <NUM_LIT:0> ] <EOL> return result <EOL> def scan_a_token ( self ) : <EOL> """<STR_LIT>""" <EOL> self . start_pos = self . cur_pos <EOL> self . start_line = self . cur_line <EOL> self . start_col = self . cur_pos - self . cur_line_start <EOL> action = self . run_machine_inlined ( ) <EOL> if action : <EOL> if self . trace : <EOL> print "<STR_LIT>" , action , "<STR_LIT>" % ( <EOL> self . start_pos , self . cur_pos ) <EOL> base = self . buf_start_pos <EOL> text = self . buffer [ self . start_pos - base : self . cur_pos - base ] <EOL> return ( text , action ) <EOL> else : <EOL> if self . cur_pos == self . start_pos : <EOL> if self . cur_char == EOL : <EOL> self . next_char ( ) <EOL> if not self . cur_char or self . cur_char == EOF : <EOL> return ( '<STR_LIT>' , None ) <EOL> raise Errors . UnrecognizedInput ( self , self . state_name ) <EOL> def run_machine ( self ) : <EOL> """<STR_LIT>""" <EOL> self . state = self . initial_state <EOL> self . backup_state = None <EOL> while self . transition ( ) : <EOL> pass <EOL> return self . back_up ( ) <EOL> def run_machine_inlined ( self ) : <EOL> """<STR_LIT>""" <EOL> state = self . initial_state <EOL> cur_pos = self . cur_pos <EOL> cur_line = self . cur_line <EOL> cur_line_start = self . cur_line_start <EOL> cur_char = self . cur_char <EOL> input_state = self . input_state <EOL> next_pos = self . next_pos <EOL> buffer = self . buffer <EOL> buf_start_pos = self . buf_start_pos <EOL> buf_len = len ( buffer ) <EOL> backup_state = None <EOL> trace = self . trace <EOL> while <NUM_LIT:1> : <EOL> if trace : <EOL> print "<STR_LIT>" % ( <EOL> state [ '<STR_LIT>' ] , input_state , cur_pos , repr ( cur_char ) ) , <EOL> action = state [ '<STR_LIT:action>' ] <EOL> if action : <EOL> backup_state = ( <EOL> action , cur_pos , cur_line , cur_line_start , cur_char , input_state , next_pos ) <EOL> c = cur_char <EOL> new_state = state . get ( c , - <NUM_LIT:1> ) <EOL> if new_state == - <NUM_LIT:1> : <EOL> new_state = c and state . get ( '<STR_LIT>' ) <EOL> if new_state : <EOL> if trace : <EOL> print "<STR_LIT>" % new_state [ '<STR_LIT>' ] <EOL> state = new_state <EOL> if input_state == <NUM_LIT:1> : <EOL> cur_pos = next_pos <EOL> buf_index = next_pos - buf_start_pos <EOL> if buf_index < buf_len : <EOL> c = buffer [ buf_index ] <EOL> next_pos = next_pos + <NUM_LIT:1> <EOL> else : <EOL> discard = self . start_pos - buf_start_pos <EOL> data = self . stream . read ( <NUM_LIT> ) <EOL> buffer = self . buffer [ discard : ] + data <EOL> self . buffer = buffer <EOL> buf_start_pos = buf_start_pos + discard <EOL> self . buf_start_pos = buf_start_pos <EOL> buf_len = len ( buffer ) <EOL> buf_index = buf_index - discard <EOL> if data : <EOL> c = buffer [ buf_index ] <EOL> next_pos = next_pos + <NUM_LIT:1> <EOL> else : <EOL> c = '<STR_LIT>' <EOL> if c == '<STR_LIT:\n>' : <EOL> cur_char = EOL <EOL> input_state = <NUM_LIT:2> <EOL> elif not c : <EOL> cur_char = EOL <EOL> input_state = <NUM_LIT:4> <EOL> else : <EOL> cur_char = c <EOL> elif input_state == <NUM_LIT:2> : <EOL> cur_char = '<STR_LIT:\n>' <EOL> input_state = <NUM_LIT:3> <EOL> elif input_state == <NUM_LIT:3> : <EOL> cur_line = cur_line + <NUM_LIT:1> <EOL> cur_line_start = cur_pos = next_pos <EOL> cur_char = BOL <EOL> input_state = <NUM_LIT:1> <EOL> elif input_state == <NUM_LIT:4> : <EOL> cur_char = EOF <EOL> input_state = <NUM_LIT:5> <EOL> else : <EOL> cur_char = '<STR_LIT>' <EOL> else : <EOL> if trace : <EOL> print "<STR_LIT>" <EOL> if backup_state : <EOL> ( action , cur_pos , cur_line , cur_line_start , <EOL> cur_char , input_state , next_pos ) = backup_state <EOL> else : <EOL> action = None <EOL> break <EOL> self . cur_pos = cur_pos <EOL> self . cur_line = cur_line <EOL> self . cur_line_start = cur_line_start <EOL> self . cur_char = cur_char <EOL> self . input_state = input_state <EOL> self . next_pos = next_pos <EOL> if trace : <EOL> if action : <EOL> print "<STR_LIT>" , action <EOL> return action <EOL> def next_char ( self ) : <EOL> input_state = self . input_state <EOL> if self . trace : <EOL> print "<STR_LIT>" , "<STR_LIT:U+0020>" * <NUM_LIT:20> , "<STR_LIT>" % ( input_state , self . cur_pos ) , <EOL> if input_state == <NUM_LIT:1> : <EOL> self . cur_pos = self . next_pos <EOL> c = self . read_char ( ) <EOL> if c == '<STR_LIT:\n>' : <EOL> self . cur_char = EOL <EOL> self . input_state = <NUM_LIT:2> <EOL> elif not c : <EOL> self . cur_char = EOL <EOL> self . input_state = <NUM_LIT:4> <EOL> else : <EOL> self . cur_char = c <EOL> elif input_state == <NUM_LIT:2> : <EOL> self . cur_char = '<STR_LIT:\n>' <EOL> self . input_state = <NUM_LIT:3> <EOL> elif input_state == <NUM_LIT:3> : <EOL> self . cur_line = self . cur_line + <NUM_LIT:1> <EOL> self . cur_line_start = self . cur_pos = self . next_pos <EOL> self . cur_char = BOL <EOL> self . input_state = <NUM_LIT:1> <EOL> elif input_state == <NUM_LIT:4> : <EOL> self . cur_char = EOF <EOL> self . input_state = <NUM_LIT:5> <EOL> else : <EOL> self . cur_char = '<STR_LIT>' <EOL> if self . trace : <EOL> print "<STR_LIT>" % ( input_state , self . cur_pos , repr ( self . cur_char ) ) <EOL> def position ( self ) : <EOL> """<STR_LIT>""" <EOL> return ( self . name , self . start_line , self . start_col ) <EOL> def begin ( self , state_name ) : <EOL> """<STR_LIT>""" <EOL> self . initial_state = ( <EOL> self . lexicon . get_initial_state ( state_name ) ) <EOL> self . state_name = state_name <EOL> def produce ( self , value , text = None ) : <EOL> """<STR_LIT>""" <EOL> if text is None : <EOL> text = self . text <EOL> self . queue . append ( ( value , text ) ) <EOL> def eof ( self ) : <EOL> """<STR_LIT>""" <EOL> setattr ( Scanner , "<STR_LIT>" , Scanner . produce ) </s>
