<s> import ctypes <EOL> from cStringIO import StringIO as BytesIO <EOL> from yajs . lib import yajl <EOL> yajl_tok_bool , yajl_tok_colon , yajl_tok_comma , yajl_tok_eof , yajl_tok_error , yajl_tok_left_bracket , yajl_tok_left_brace , yajl_tok_null , yajl_tok_right_bracket , yajl_tok_right_brace , yajl_tok_integer , yajl_tok_double , yajl_tok_string , yajl_tok_string_with_escapes , yajl_tok_comment = range ( <NUM_LIT:15> ) <EOL> tokens_want_value = frozenset ( [ <EOL> yajl_tok_bool , <EOL> yajl_tok_integer , <EOL> yajl_tok_double , <EOL> yajl_tok_string , <EOL> ] ) <EOL> token_value_converters = { <EOL> yajl_tok_bool : lambda x : x == '<STR_LIT:true>' , <EOL> yajl_tok_integer : int , <EOL> yajl_tok_double : float , <EOL> yajl_tok_string : lambda x : x . decode ( '<STR_LIT:utf-8>' ) , <EOL> yajl_tok_null : lambda x : None , <EOL> } <EOL> yajl_alloc_func_buffer = ctypes . c_void_p * <NUM_LIT:4> <EOL> yajl_lex_alloc = yajl . yajl_lex_alloc <EOL> yajl_lex_alloc . argtypes = ( ctypes . c_void_p , ctypes . c_uint , ctypes . c_uint ) <EOL> yajl_lex_alloc . restype = ctypes . c_void_p <EOL> yajl_lex_free = yajl . yajl_lex_free <EOL> yajl_lex_free . argtypes = ( ctypes . c_void_p , ) <EOL> yajl_lex_lex = yajl . yajl_lex_lex <EOL> yajl_lex_lex . argtypes = ( ctypes . c_void_p , ctypes . c_char_p , <EOL> ctypes . c_size_t , ctypes . POINTER ( ctypes . c_size_t ) , <EOL> ctypes . POINTER ( ctypes . c_char_p ) , <EOL> ctypes . POINTER ( ctypes . c_size_t ) ) <EOL> yajl_set_default_alloc_funcs = yajl . yajl_set_default_alloc_funcs <EOL> yajl_set_default_alloc_funcs . argtypes = ( ctypes . c_void_p , ) <EOL> yajl_buf_alloc = yajl . yajl_buf_alloc <EOL> yajl_buf_alloc . argtypes = ( ctypes . c_void_p , ) <EOL> yajl_buf_alloc . restype = ctypes . c_void_p <EOL> yajl_buf_free = yajl . yajl_buf_free <EOL> yajl_buf_free . argtypes = ( ctypes . c_void_p , ) <EOL> yajl_buf_data = yajl . yajl_buf_data <EOL> yajl_buf_data . restype = ctypes . c_char_p <EOL> yajl_buf_data . argtypes = ( ctypes . c_void_p , ) <EOL> yajl_buf_len = yajl . yajl_buf_len <EOL> yajl_buf_len . restype = ctypes . c_size_t <EOL> yajl_buf_len . argtypes = ( ctypes . c_void_p , ) <EOL> yajl_buf_clear = yajl . yajl_buf_clear <EOL> yajl_buf_clear . argtypes = ( ctypes . c_void_p , ) <EOL> yajl_string_decode = yajl . yajl_string_decode <EOL> yajl_string_decode . argtypes = ( ctypes . c_void_p , ctypes . c_char_p , <EOL> ctypes . c_size_t ) <EOL> def _ll_tokenize ( chunk_iter , allow_comments ) : <EOL> """<STR_LIT>""" <EOL> alloc_funcs = yajl_alloc_func_buffer ( ) <EOL> yajl_set_default_alloc_funcs ( ctypes . byref ( alloc_funcs ) ) <EOL> lexer = yajl_lex_alloc ( ctypes . byref ( alloc_funcs ) , allow_comments , False ) <EOL> decode_buffer = yajl_buf_alloc ( ctypes . byref ( alloc_funcs ) ) <EOL> try : <EOL> out_buffer = ctypes . c_char_p ( ) <EOL> out_buffer_ref = ctypes . byref ( out_buffer ) <EOL> out_len = ctypes . c_size_t ( ) <EOL> out_len_ref = ctypes . byref ( out_len ) <EOL> for chunk in chunk_iter : <EOL> chunk_p = ctypes . c_char_p ( chunk ) <EOL> chunk_len = len ( chunk ) <EOL> offset = ctypes . c_size_t ( <NUM_LIT:0> ) <EOL> offset_ref = ctypes . byref ( offset ) <EOL> while <NUM_LIT:1> : <EOL> tok = yajl_lex_lex ( lexer , chunk_p , chunk_len , <EOL> offset_ref , <EOL> out_buffer_ref , <EOL> out_len_ref ) <EOL> if tok == yajl_tok_eof : <EOL> break <EOL> elif tok == yajl_tok_error : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> elif tok == yajl_tok_comment : <EOL> continue <EOL> elif tok == yajl_tok_string_with_escapes : <EOL> yajl_string_decode ( decode_buffer , <EOL> out_buffer , out_len . value ) <EOL> value = ctypes . string_at ( yajl_buf_data ( decode_buffer ) , <EOL> yajl_buf_len ( decode_buffer ) ) <EOL> yajl_buf_clear ( decode_buffer ) <EOL> tok = yajl_tok_string <EOL> elif tok in tokens_want_value : <EOL> value = ctypes . string_at ( out_buffer , out_len . value ) <EOL> else : <EOL> value = None <EOL> yield tok , value <EOL> finally : <EOL> yajl_lex_free ( lexer ) <EOL> yajl_buf_free ( decode_buffer ) <EOL> def tokenize ( f , allow_comments = False , buffer_size = <NUM_LIT:8> * <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> def _iter_chunks ( ) : <EOL> while <NUM_LIT:1> : <EOL> line = f . readline ( buffer_size ) <EOL> if not line : <EOL> break <EOL> yield line <EOL> yield '<STR_LIT:U+0020>' <EOL> def _build ( token , value ) : <EOL> if token == yajl_tok_left_brace : <EOL> yield '<STR_LIT>' , None <EOL> first = True <EOL> while <NUM_LIT:1> : <EOL> token , value = tokeniter . next ( ) <EOL> if token == yajl_tok_right_brace : <EOL> break <EOL> if not first : <EOL> if token != yajl_tok_comma : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> token , value = tokeniter . next ( ) <EOL> first = False <EOL> for event in _build ( token , value ) : <EOL> yield event <EOL> token , _ = tokeniter . next ( ) <EOL> if token != yajl_tok_colon : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> for event in _build ( * tokeniter . next ( ) ) : <EOL> yield event <EOL> yield '<STR_LIT>' , None <EOL> elif token == yajl_tok_left_bracket : <EOL> yield '<STR_LIT>' , None <EOL> first = True <EOL> while <NUM_LIT:1> : <EOL> token , value = tokeniter . next ( ) <EOL> if token == yajl_tok_right_bracket : <EOL> break <EOL> if not first : <EOL> if token != yajl_tok_comma : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> token , value = tokeniter . next ( ) <EOL> first = False <EOL> for event in _build ( token , value ) : <EOL> yield event <EOL> yield '<STR_LIT>' , None <EOL> else : <EOL> conv = token_value_converters . get ( token ) <EOL> if conv is None : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> yield '<STR_LIT:value>' , conv ( value ) <EOL> tokeniter = _ll_tokenize ( _iter_chunks ( ) , allow_comments ) <EOL> try : <EOL> first = tokeniter . next ( ) <EOL> except StopIteration : <EOL> return iter ( ( ) ) <EOL> return _build ( * first ) <EOL> def tokenize_string ( string , allow_comments = False ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( string , unicode ) : <EOL> string = string . encode ( '<STR_LIT:utf-8>' ) <EOL> f = BytesIO ( string ) <EOL> return tokenize ( f , allow_comments ) </s>
