<s> import json <EOL> from six import unichr <EOL> from . error import LanguageError <EOL> __all__ = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] <EOL> class Token ( object ) : <EOL> __slots__ = '<STR_LIT>' , '<STR_LIT:start>' , '<STR_LIT:end>' , '<STR_LIT:value>' <EOL> def __init__ ( self , kind , start , end , value = None ) : <EOL> self . kind = kind <EOL> self . start = start <EOL> self . end = end <EOL> self . value = value <EOL> def __repr__ ( self ) : <EOL> return u'<STR_LIT>' . format ( <EOL> get_token_kind_desc ( self . kind ) , <EOL> self . start , <EOL> self . end , <EOL> repr ( self . value ) <EOL> ) <EOL> def __eq__ ( self , other ) : <EOL> return ( self . kind == other . kind and <EOL> self . start == other . start and <EOL> self . end == other . end and <EOL> self . value == other . value ) <EOL> class Lexer ( object ) : <EOL> __slots__ = '<STR_LIT:source>' , '<STR_LIT>' <EOL> def __init__ ( self , source ) : <EOL> self . source = source <EOL> self . prev_position = <NUM_LIT:0> <EOL> def next_token ( self , reset_position = None ) : <EOL> if reset_position is None : <EOL> reset_position = self . prev_position <EOL> token = read_token ( self . source , reset_position ) <EOL> self . prev_position = token . end <EOL> return token <EOL> class TokenKind ( object ) : <EOL> EOF = <NUM_LIT:1> <EOL> BANG = <NUM_LIT:2> <EOL> DOLLAR = <NUM_LIT:3> <EOL> PAREN_L = <NUM_LIT:4> <EOL> PAREN_R = <NUM_LIT:5> <EOL> SPREAD = <NUM_LIT:6> <EOL> COLON = <NUM_LIT:7> <EOL> EQUALS = <NUM_LIT:8> <EOL> AT = <NUM_LIT:9> <EOL> BRACKET_L = <NUM_LIT:10> <EOL> BRACKET_R = <NUM_LIT:11> <EOL> BRACE_L = <NUM_LIT:12> <EOL> PIPE = <NUM_LIT> <EOL> BRACE_R = <NUM_LIT> <EOL> NAME = <NUM_LIT:15> <EOL> VARIABLE = <NUM_LIT:16> <EOL> INT = <NUM_LIT> <EOL> FLOAT = <NUM_LIT> <EOL> STRING = <NUM_LIT> <EOL> def get_token_desc ( token ) : <EOL> if token . value : <EOL> return u'<STR_LIT>' . format ( <EOL> get_token_kind_desc ( token . kind ) , <EOL> token . value <EOL> ) <EOL> else : <EOL> return get_token_kind_desc ( token . kind ) <EOL> def get_token_kind_desc ( kind ) : <EOL> return TOKEN_DESCRIPTION [ kind ] <EOL> TOKEN_DESCRIPTION = { <EOL> TokenKind . EOF : '<STR_LIT>' , <EOL> TokenKind . BANG : '<STR_LIT:!>' , <EOL> TokenKind . DOLLAR : '<STR_LIT:$>' , <EOL> TokenKind . PAREN_L : '<STR_LIT:(>' , <EOL> TokenKind . PAREN_R : '<STR_LIT:)>' , <EOL> TokenKind . SPREAD : '<STR_LIT>' , <EOL> TokenKind . COLON : '<STR_LIT::>' , <EOL> TokenKind . EQUALS : '<STR_LIT:=>' , <EOL> TokenKind . AT : '<STR_LIT:@>' , <EOL> TokenKind . BRACKET_L : '<STR_LIT:[>' , <EOL> TokenKind . BRACKET_R : '<STR_LIT:]>' , <EOL> TokenKind . BRACE_L : '<STR_LIT:{>' , <EOL> TokenKind . PIPE : '<STR_LIT:|>' , <EOL> TokenKind . BRACE_R : '<STR_LIT:}>' , <EOL> TokenKind . NAME : '<STR_LIT:Name>' , <EOL> TokenKind . VARIABLE : '<STR_LIT>' , <EOL> TokenKind . INT : '<STR_LIT>' , <EOL> TokenKind . FLOAT : '<STR_LIT>' , <EOL> TokenKind . STRING : '<STR_LIT>' , <EOL> } <EOL> def char_code_at ( s , pos ) : <EOL> if <NUM_LIT:0> <= pos < len ( s ) : <EOL> return ord ( s [ pos ] ) <EOL> return None <EOL> PUNCT_CODE_TO_KIND = { <EOL> ord ( '<STR_LIT:!>' ) : TokenKind . BANG , <EOL> ord ( '<STR_LIT:$>' ) : TokenKind . DOLLAR , <EOL> ord ( '<STR_LIT:(>' ) : TokenKind . PAREN_L , <EOL> ord ( '<STR_LIT:)>' ) : TokenKind . PAREN_R , <EOL> ord ( '<STR_LIT::>' ) : TokenKind . COLON , <EOL> ord ( '<STR_LIT:=>' ) : TokenKind . EQUALS , <EOL> ord ( '<STR_LIT:@>' ) : TokenKind . AT , <EOL> ord ( '<STR_LIT:[>' ) : TokenKind . BRACKET_L , <EOL> ord ( '<STR_LIT:]>' ) : TokenKind . BRACKET_R , <EOL> ord ( '<STR_LIT:{>' ) : TokenKind . BRACE_L , <EOL> ord ( '<STR_LIT:|>' ) : TokenKind . PIPE , <EOL> ord ( '<STR_LIT:}>' ) : TokenKind . BRACE_R , <EOL> } <EOL> def print_char_code ( code ) : <EOL> if code is None : <EOL> return '<STR_LIT>' <EOL> if code < <NUM_LIT> : <EOL> return json . dumps ( unichr ( code ) ) <EOL> return '<STR_LIT>' % code <EOL> def read_token ( source , from_position ) : <EOL> """<STR_LIT>""" <EOL> body = source . body <EOL> body_length = len ( body ) <EOL> position = position_after_whitespace ( body , from_position ) <EOL> if position >= body_length : <EOL> return Token ( TokenKind . EOF , position , position ) <EOL> code = char_code_at ( body , position ) <EOL> if code < <NUM_LIT> and code not in ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) : <EOL> raise LanguageError ( <EOL> source , position , <EOL> u'<STR_LIT>' . format ( print_char_code ( code ) ) <EOL> ) <EOL> kind = PUNCT_CODE_TO_KIND . get ( code ) <EOL> if kind is not None : <EOL> return Token ( kind , position , position + <NUM_LIT:1> ) <EOL> if code == <NUM_LIT> : <EOL> if char_code_at ( body , position + <NUM_LIT:1> ) == char_code_at ( body , position + <NUM_LIT:2> ) == <NUM_LIT> : <EOL> return Token ( TokenKind . SPREAD , position , position + <NUM_LIT:3> ) <EOL> elif <NUM_LIT> <= code <= <NUM_LIT> or code == <NUM_LIT> or <NUM_LIT> <= code <= <NUM_LIT> : <EOL> return read_name ( source , position ) <EOL> elif code == <NUM_LIT> or <NUM_LIT> <= code <= <NUM_LIT> : <EOL> return read_number ( source , position , code ) <EOL> elif code == <NUM_LIT> : <EOL> return read_string ( source , position ) <EOL> raise LanguageError ( <EOL> source , position , <EOL> u'<STR_LIT>' . format ( print_char_code ( code ) ) ) <EOL> ignored_whitespace_characters = frozenset ( [ <EOL> <NUM_LIT> , <EOL> <NUM_LIT> , <EOL> <NUM_LIT> , <EOL> <NUM_LIT> , <EOL> <NUM_LIT> , <EOL> <NUM_LIT> <EOL> ] ) <EOL> def position_after_whitespace ( body , start_position ) : <EOL> """<STR_LIT>""" <EOL> body_length = len ( body ) <EOL> position = start_position <EOL> while position < body_length : <EOL> code = char_code_at ( body , position ) <EOL> if code in ignored_whitespace_characters : <EOL> position += <NUM_LIT:1> <EOL> elif code == <NUM_LIT> : <EOL> position += <NUM_LIT:1> <EOL> while position < body_length : <EOL> code = char_code_at ( body , position ) <EOL> if not ( code is not None and ( code > <NUM_LIT> or code == <NUM_LIT> ) and code not in ( <NUM_LIT> , <NUM_LIT> ) ) : <EOL> break <EOL> position += <NUM_LIT:1> <EOL> else : <EOL> break <EOL> return position <EOL> def read_number ( source , start , first_code ) : <EOL> """<STR_LIT>""" <EOL> code = first_code <EOL> body = source . body <EOL> position = start <EOL> is_float = False <EOL> if code == <NUM_LIT> : <EOL> position += <NUM_LIT:1> <EOL> code = char_code_at ( body , position ) <EOL> if code == <NUM_LIT> : <EOL> position += <NUM_LIT:1> <EOL> code = char_code_at ( body , position ) <EOL> if code is not None and <NUM_LIT> <= code <= <NUM_LIT> : <EOL> raise LanguageError ( <EOL> source , <EOL> position , <EOL> u'<STR_LIT>' . format ( print_char_code ( code ) ) <EOL> ) <EOL> else : <EOL> position = read_digits ( source , position , code ) <EOL> code = char_code_at ( body , position ) <EOL> if code == <NUM_LIT> : <EOL> is_float = True <EOL> position += <NUM_LIT:1> <EOL> code = char_code_at ( body , position ) <EOL> position = read_digits ( source , position , code ) <EOL> code = char_code_at ( body , position ) <EOL> if code in ( <NUM_LIT> , <NUM_LIT> ) : <EOL> is_float = True <EOL> position += <NUM_LIT:1> <EOL> code = char_code_at ( body , position ) <EOL> if code in ( <NUM_LIT> , <NUM_LIT> ) : <EOL> position += <NUM_LIT:1> <EOL> code = char_code_at ( body , position ) <EOL> position = read_digits ( source , position , code ) <EOL> return Token ( <EOL> TokenKind . FLOAT if is_float else TokenKind . INT , <EOL> start , <EOL> position , <EOL> body [ start : position ] <EOL> ) <EOL> def read_digits ( source , start , first_code ) : <EOL> body = source . body <EOL> position = start <EOL> code = first_code <EOL> if code is not None and <NUM_LIT> <= code <= <NUM_LIT> : <EOL> while True : <EOL> position += <NUM_LIT:1> <EOL> code = char_code_at ( body , position ) <EOL> if not ( code is not None and <NUM_LIT> <= code <= <NUM_LIT> ) : <EOL> break <EOL> return position <EOL> raise LanguageError ( <EOL> source , <EOL> position , <EOL> u'<STR_LIT>' . format ( print_char_code ( code ) ) <EOL> ) <EOL> ESCAPED_CHAR_CODES = { <EOL> <NUM_LIT> : '<STR_LIT:">' , <EOL> <NUM_LIT> : '<STR_LIT:/>' , <EOL> <NUM_LIT> : '<STR_LIT:\\>' , <EOL> <NUM_LIT> : '<STR_LIT>' , <EOL> <NUM_LIT> : '<STR_LIT>' , <EOL> <NUM_LIT> : '<STR_LIT:\n>' , <EOL> <NUM_LIT> : '<STR_LIT:\r>' , <EOL> <NUM_LIT> : '<STR_LIT:\t>' , <EOL> } <EOL> def read_string ( source , start ) : <EOL> """<STR_LIT>""" <EOL> body = source . body <EOL> body_length = len ( body ) <EOL> position = start + <NUM_LIT:1> <EOL> chunk_start = position <EOL> code = <NUM_LIT:0> <EOL> value = [ ] <EOL> append = value . append <EOL> while position < body_length : <EOL> code = char_code_at ( body , position ) <EOL> if not ( <EOL> code is not None and <EOL> code not in ( <EOL> <NUM_LIT> , <NUM_LIT> , <EOL> <NUM_LIT> <EOL> ) <EOL> ) : <EOL> break <EOL> if code < <NUM_LIT> and code != <NUM_LIT> : <EOL> raise LanguageError ( <EOL> source , <EOL> position , <EOL> u'<STR_LIT>' . format ( print_char_code ( code ) ) <EOL> ) <EOL> position += <NUM_LIT:1> <EOL> if code == <NUM_LIT> : <EOL> append ( body [ chunk_start : position - <NUM_LIT:1> ] ) <EOL> code = char_code_at ( body , position ) <EOL> escaped = ESCAPED_CHAR_CODES . get ( code ) <EOL> if escaped is not None : <EOL> append ( escaped ) <EOL> elif code == <NUM_LIT> : <EOL> char_code = uni_char_code ( <EOL> char_code_at ( body , position + <NUM_LIT:1> ) or <NUM_LIT:0> , <EOL> char_code_at ( body , position + <NUM_LIT:2> ) or <NUM_LIT:0> , <EOL> char_code_at ( body , position + <NUM_LIT:3> ) or <NUM_LIT:0> , <EOL> char_code_at ( body , position + <NUM_LIT:4> ) or <NUM_LIT:0> , <EOL> ) <EOL> if char_code < <NUM_LIT:0> : <EOL> raise LanguageError ( <EOL> source , position , <EOL> u'<STR_LIT>' . format ( body [ position + <NUM_LIT:1> : position + <NUM_LIT:5> ] ) <EOL> ) <EOL> append ( unichr ( char_code ) ) <EOL> position += <NUM_LIT:4> <EOL> else : <EOL> raise LanguageError ( <EOL> source , position , <EOL> u'<STR_LIT>' . format ( unichr ( code ) ) <EOL> ) <EOL> position += <NUM_LIT:1> <EOL> chunk_start = position <EOL> if code != <NUM_LIT> : <EOL> raise LanguageError ( source , position , '<STR_LIT>' ) <EOL> append ( body [ chunk_start : position ] ) <EOL> return Token ( TokenKind . STRING , start , position + <NUM_LIT:1> , u'<STR_LIT>' . join ( value ) ) <EOL> def uni_char_code ( a , b , c , d ) : <EOL> """<STR_LIT>""" <EOL> return ( char2hex ( a ) << <NUM_LIT:12> | char2hex ( b ) << <NUM_LIT:8> | <EOL> char2hex ( c ) << <NUM_LIT:4> | char2hex ( d ) ) <EOL> def char2hex ( a ) : <EOL> """<STR_LIT>""" <EOL> if <NUM_LIT> <= a <= <NUM_LIT> : <EOL> return a - <NUM_LIT> <EOL> elif <NUM_LIT> <= a <= <NUM_LIT> : <EOL> return a - <NUM_LIT> <EOL> elif <NUM_LIT> <= a <= <NUM_LIT> : <EOL> return a - <NUM_LIT> <EOL> return - <NUM_LIT:1> <EOL> def read_name ( source , position ) : <EOL> """<STR_LIT>""" <EOL> body = source . body <EOL> body_length = len ( body ) <EOL> end = position + <NUM_LIT:1> <EOL> while end != body_length : <EOL> code = char_code_at ( body , end ) <EOL> if not ( code is not None and ( <EOL> code == <NUM_LIT> or <EOL> <NUM_LIT> <= code <= <NUM_LIT> or <EOL> <NUM_LIT> <= code <= <NUM_LIT> or <EOL> <NUM_LIT> <= code <= <NUM_LIT> <EOL> ) ) : <EOL> break <EOL> end += <NUM_LIT:1> <EOL> return Token ( TokenKind . NAME , position , end , body [ position : end ] ) </s>
