<s> """<STR_LIT>""" <EOL> class _TokenType ( tuple ) : <EOL> parent = None <EOL> def split ( self ) : <EOL> buf = [ ] <EOL> node = self <EOL> while node is not None : <EOL> buf . append ( node ) <EOL> node = node . parent <EOL> buf . reverse ( ) <EOL> return buf <EOL> def __init__ ( self , * args ) : <EOL> self . subtypes = set ( ) <EOL> def __contains__ ( self , val ) : <EOL> return self is val or ( <EOL> type ( val ) is self . __class__ and <EOL> val [ : len ( self ) ] == self <EOL> ) <EOL> def __getattr__ ( self , val ) : <EOL> if not val or not val [ <NUM_LIT:0> ] . isupper ( ) : <EOL> return tuple . __getattribute__ ( self , val ) <EOL> new = _TokenType ( self + ( val , ) ) <EOL> setattr ( self , val , new ) <EOL> self . subtypes . add ( new ) <EOL> new . parent = self <EOL> return new <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' + ( self and '<STR_LIT:.>' or '<STR_LIT>' ) + '<STR_LIT:.>' . join ( self ) <EOL> Token = _TokenType ( ) <EOL> Text = Token . Text <EOL> Whitespace = Text . Whitespace <EOL> Error = Token . Error <EOL> Other = Token . Other <EOL> Keyword = Token . Keyword <EOL> Name = Token . Name <EOL> Literal = Token . Literal <EOL> String = Literal . String <EOL> Number = Literal . Number <EOL> Punctuation = Token . Punctuation <EOL> Operator = Token . Operator <EOL> Comment = Token . Comment <EOL> Generic = Token . Generic <EOL> Token . Token = Token <EOL> Token . String = String <EOL> Token . Number = Number <EOL> def is_token_subtype ( ttype , other ) : <EOL> """<STR_LIT>""" <EOL> return ttype in other <EOL> def string_to_tokentype ( s ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( s , _TokenType ) : <EOL> return s <EOL> if not s : <EOL> return Token <EOL> node = Token <EOL> for item in s . split ( '<STR_LIT:.>' ) : <EOL> node = getattr ( node , item ) <EOL> return node <EOL> STANDARD_TYPES = { <EOL> Token : '<STR_LIT>' , <EOL> Text : '<STR_LIT>' , <EOL> Whitespace : '<STR_LIT:w>' , <EOL> Error : '<STR_LIT>' , <EOL> Other : '<STR_LIT:x>' , <EOL> Keyword : '<STR_LIT:k>' , <EOL> Keyword . Constant : '<STR_LIT>' , <EOL> Keyword . Declaration : '<STR_LIT>' , <EOL> Keyword . Namespace : '<STR_LIT>' , <EOL> Keyword . Pseudo : '<STR_LIT>' , <EOL> Keyword . Reserved : '<STR_LIT>' , <EOL> Keyword . Type : '<STR_LIT>' , <EOL> Name : '<STR_LIT:n>' , <EOL> Name . Attribute : '<STR_LIT>' , <EOL> Name . Builtin : '<STR_LIT>' , <EOL> Name . Builtin . Pseudo : '<STR_LIT>' , <EOL> Name . Class : '<STR_LIT>' , <EOL> Name . Constant : '<STR_LIT>' , <EOL> Name . Decorator : '<STR_LIT>' , <EOL> Name . Entity : '<STR_LIT>' , <EOL> Name . Exception : '<STR_LIT>' , <EOL> Name . Function : '<STR_LIT>' , <EOL> Name . Property : '<STR_LIT>' , <EOL> Name . Label : '<STR_LIT>' , <EOL> Name . Namespace : '<STR_LIT>' , <EOL> Name . Other : '<STR_LIT>' , <EOL> Name . Tag : '<STR_LIT>' , <EOL> Name . Variable : '<STR_LIT>' , <EOL> Name . Variable . Class : '<STR_LIT>' , <EOL> Name . Variable . Global : '<STR_LIT>' , <EOL> Name . Variable . Instance : '<STR_LIT>' , <EOL> Literal : '<STR_LIT:l>' , <EOL> Literal . Date : '<STR_LIT>' , <EOL> String : '<STR_LIT:s>' , <EOL> String . Backtick : '<STR_LIT>' , <EOL> String . Char : '<STR_LIT>' , <EOL> String . Doc : '<STR_LIT>' , <EOL> String . Double : '<STR_LIT>' , <EOL> String . Escape : '<STR_LIT>' , <EOL> String . Heredoc : '<STR_LIT>' , <EOL> String . Interpol : '<STR_LIT>' , <EOL> String . Other : '<STR_LIT>' , <EOL> String . Regex : '<STR_LIT>' , <EOL> String . Single : '<STR_LIT>' , <EOL> String . Symbol : '<STR_LIT>' , <EOL> Number : '<STR_LIT:m>' , <EOL> Number . Float : '<STR_LIT>' , <EOL> Number . Hex : '<STR_LIT>' , <EOL> Number . Integer : '<STR_LIT>' , <EOL> Number . Integer . Long : '<STR_LIT>' , <EOL> Number . Oct : '<STR_LIT>' , <EOL> Operator : '<STR_LIT:o>' , <EOL> Operator . Word : '<STR_LIT>' , <EOL> Punctuation : '<STR_LIT:p>' , <EOL> Comment : '<STR_LIT:c>' , <EOL> Comment . Multiline : '<STR_LIT>' , <EOL> Comment . Preproc : '<STR_LIT>' , <EOL> Comment . Single : '<STR_LIT>' , <EOL> Comment . Special : '<STR_LIT>' , <EOL> Generic : '<STR_LIT:g>' , <EOL> Generic . Deleted : '<STR_LIT>' , <EOL> Generic . Emph : '<STR_LIT>' , <EOL> Generic . Error : '<STR_LIT>' , <EOL> Generic . Heading : '<STR_LIT>' , <EOL> Generic . Inserted : '<STR_LIT>' , <EOL> Generic . Output : '<STR_LIT>' , <EOL> Generic . Prompt : '<STR_LIT>' , <EOL> Generic . Strong : '<STR_LIT>' , <EOL> Generic . Subheading : '<STR_LIT>' , <EOL> Generic . Traceback : '<STR_LIT>' , <EOL> } </s>
