<s> """<STR_LIT>""" <EOL> import sre_parse , sre_compile , sre_constants <EOL> from sre_constants import BRANCH , SUBPATTERN <EOL> from sre import VERBOSE , MULTILINE , DOTALL <EOL> import re <EOL> __all__ = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> FLAGS = ( VERBOSE | MULTILINE | DOTALL ) <EOL> class Scanner ( object ) : <EOL> def __init__ ( self , lexicon , flags = FLAGS ) : <EOL> self . actions = [ None ] <EOL> s = sre_parse . Pattern ( ) <EOL> s . flags = flags <EOL> p = [ ] <EOL> for idx , token in enumerate ( lexicon ) : <EOL> phrase = token . pattern <EOL> try : <EOL> subpattern = sre_parse . SubPattern ( s , <EOL> [ ( SUBPATTERN , ( idx + <NUM_LIT:1> , sre_parse . parse ( phrase , flags ) ) ) ] ) <EOL> except sre_constants . error : <EOL> raise <EOL> p . append ( subpattern ) <EOL> self . actions . append ( token ) <EOL> p = sre_parse . SubPattern ( s , [ ( BRANCH , ( None , p ) ) ] ) <EOL> self . scanner = sre_compile . compile ( p ) <EOL> def iterscan ( self , string , idx = <NUM_LIT:0> , context = None ) : <EOL> """<STR_LIT>""" <EOL> match = self . scanner . scanner ( string , idx ) . match <EOL> actions = self . actions <EOL> lastend = idx <EOL> end = len ( string ) <EOL> while True : <EOL> m = match ( ) <EOL> if m is None : <EOL> break <EOL> matchbegin , matchend = m . span ( ) <EOL> if lastend == matchend : <EOL> break <EOL> action = actions [ m . lastindex ] <EOL> if action is not None : <EOL> rval , next_pos = action ( m , context ) <EOL> if next_pos is not None and next_pos != matchend : <EOL> matchend = next_pos <EOL> match = self . scanner . scanner ( string , matchend ) . match <EOL> yield rval , matchend <EOL> lastend = matchend <EOL> def pattern ( pattern , flags = FLAGS ) : <EOL> def decorator ( fn ) : <EOL> fn . pattern = pattern <EOL> fn . regex = re . compile ( pattern , flags ) <EOL> return fn <EOL> return decorator </s>
