<s> import types <EOL> import Actions <EOL> import DFA <EOL> import Errors <EOL> import Machines <EOL> import Regexps <EOL> DUMP_NFA = <NUM_LIT:1> <EOL> DUMP_DFA = <NUM_LIT:2> <EOL> class State : <EOL> """<STR_LIT>""" <EOL> name = None <EOL> tokens = None <EOL> def __init__ ( self , name , tokens ) : <EOL> self . name = name <EOL> self . tokens = tokens <EOL> class Lexicon : <EOL> """<STR_LIT>""" <EOL> machine = None <EOL> tables = None <EOL> def __init__ ( self , specifications , debug = None , debug_flags = <NUM_LIT:7> , timings = None ) : <EOL> if type ( specifications ) != types . ListType : <EOL> raise Errors . InvalidScanner ( "<STR_LIT>" ) <EOL> if timings : <EOL> from Timing import time <EOL> total_time = <NUM_LIT:0.0> <EOL> time1 = time ( ) <EOL> nfa = Machines . Machine ( ) <EOL> default_initial_state = nfa . new_initial_state ( '<STR_LIT>' ) <EOL> token_number = <NUM_LIT:1> <EOL> for spec in specifications : <EOL> if isinstance ( spec , State ) : <EOL> user_initial_state = nfa . new_initial_state ( spec . name ) <EOL> for token in spec . tokens : <EOL> self . add_token_to_machine ( <EOL> nfa , user_initial_state , token , token_number ) <EOL> token_number = token_number + <NUM_LIT:1> <EOL> elif type ( spec ) == types . TupleType : <EOL> self . add_token_to_machine ( <EOL> nfa , default_initial_state , spec , token_number ) <EOL> token_number = token_number + <NUM_LIT:1> <EOL> else : <EOL> raise Errors . InvalidToken ( <EOL> token_number , <EOL> "<STR_LIT>" ) <EOL> if timings : <EOL> time2 = time ( ) <EOL> total_time = total_time + ( time2 - time1 ) <EOL> time3 = time ( ) <EOL> if debug and ( debug_flags & <NUM_LIT:1> ) : <EOL> debug . write ( "<STR_LIT>" ) <EOL> nfa . dump ( debug ) <EOL> dfa = DFA . nfa_to_dfa ( nfa , debug = ( debug_flags & <NUM_LIT:3> ) == <NUM_LIT:3> and debug ) <EOL> if timings : <EOL> time4 = time ( ) <EOL> total_time = total_time + ( time4 - time3 ) <EOL> if debug and ( debug_flags & <NUM_LIT:2> ) : <EOL> debug . write ( "<STR_LIT>" ) <EOL> dfa . dump ( debug ) <EOL> if timings : <EOL> timings . write ( "<STR_LIT>" % ( time2 - time1 ) ) <EOL> timings . write ( "<STR_LIT>" % ( time4 - time3 ) ) <EOL> timings . write ( "<STR_LIT>" % total_time ) <EOL> self . machine = dfa <EOL> def add_token_to_machine ( self , machine , initial_state , token_spec , token_number ) : <EOL> try : <EOL> ( re , action_spec ) = self . parse_token_definition ( token_spec ) <EOL> if isinstance ( action_spec , Actions . Action ) : <EOL> action = action_spec <EOL> elif callable ( action_spec ) : <EOL> action = Actions . Call ( action_spec ) <EOL> else : <EOL> action = Actions . Return ( action_spec ) <EOL> final_state = machine . new_state ( ) <EOL> re . build_machine ( machine , initial_state , final_state , <EOL> match_bol = <NUM_LIT:1> , nocase = <NUM_LIT:0> ) <EOL> final_state . set_action ( action , priority = - token_number ) <EOL> except Errors . PlexError , e : <EOL> raise e . __class__ ( "<STR_LIT>" % ( token_number , e ) ) <EOL> def parse_token_definition ( self , token_spec ) : <EOL> if type ( token_spec ) != types . TupleType : <EOL> raise Errors . InvalidToken ( "<STR_LIT>" ) <EOL> if len ( token_spec ) != <NUM_LIT:2> : <EOL> raise Errors . InvalidToken ( "<STR_LIT>" ) <EOL> pattern , action = token_spec <EOL> if not isinstance ( pattern , Regexps . RE ) : <EOL> raise Errors . InvalidToken ( "<STR_LIT>" ) <EOL> return ( pattern , action ) <EOL> def get_initial_state ( self , name ) : <EOL> return self . machine . get_initial_state ( name ) </s>
