<s> from __future__ import absolute_import <EOL> import cython <EOL> cython . declare ( make_lexicon = object , lexicon = object , <EOL> print_function = object , error = object , warning = object , <EOL> os = object , platform = object ) <EOL> import os <EOL> import platform <EOL> from . . import Utils <EOL> from . . Plex . Scanners import Scanner <EOL> from . . Plex . Errors import UnrecognizedInput <EOL> from . Errors import error , warning <EOL> from . Lexicon import any_string_prefix , make_lexicon , IDENT <EOL> from . Future import print_function <EOL> debug_scanner = <NUM_LIT:0> <EOL> trace_scanner = <NUM_LIT:0> <EOL> scanner_debug_flags = <NUM_LIT:0> <EOL> scanner_dump_file = None <EOL> lexicon = None <EOL> def get_lexicon ( ) : <EOL> global lexicon <EOL> if not lexicon : <EOL> lexicon = make_lexicon ( ) <EOL> return lexicon <EOL> py_reserved_words = [ <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:class>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> ] <EOL> pyx_reserved_words = py_reserved_words + [ <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" <EOL> ] <EOL> class Method ( object ) : <EOL> def __init__ ( self , name , ** kwargs ) : <EOL> self . name = name <EOL> self . kwargs = kwargs or None <EOL> self . __name__ = name <EOL> def __call__ ( self , stream , text ) : <EOL> method = getattr ( stream , self . name ) <EOL> return method ( text , ** self . kwargs ) if self . kwargs is not None else method ( text ) <EOL> class CompileTimeScope ( object ) : <EOL> def __init__ ( self , outer = None ) : <EOL> self . entries = { } <EOL> self . outer = outer <EOL> def declare ( self , name , value ) : <EOL> self . entries [ name ] = value <EOL> def update ( self , other ) : <EOL> self . entries . update ( other ) <EOL> def lookup_here ( self , name ) : <EOL> return self . entries [ name ] <EOL> def __contains__ ( self , name ) : <EOL> return name in self . entries <EOL> def lookup ( self , name ) : <EOL> try : <EOL> return self . lookup_here ( name ) <EOL> except KeyError : <EOL> outer = self . outer <EOL> if outer : <EOL> return outer . lookup ( name ) <EOL> else : <EOL> raise <EOL> def initial_compile_time_env ( ) : <EOL> benv = CompileTimeScope ( ) <EOL> names = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> for name , value in zip ( names , platform . uname ( ) ) : <EOL> benv . declare ( name , value ) <EOL> try : <EOL> import __builtin__ as builtins <EOL> except ImportError : <EOL> import builtins <EOL> names = ( <EOL> '<STR_LIT:False>' , '<STR_LIT:True>' , <EOL> '<STR_LIT>' , '<STR_LIT:all>' , '<STR_LIT>' , '<STR_LIT:ascii>' , '<STR_LIT>' , '<STR_LIT:bool>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT:float>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:int>' , '<STR_LIT>' , <EOL> '<STR_LIT:list>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:str>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> ) <EOL> for name in names : <EOL> try : <EOL> benv . declare ( name , getattr ( builtins , name ) ) <EOL> except AttributeError : <EOL> pass <EOL> from functools import reduce <EOL> benv . declare ( '<STR_LIT>' , reduce ) <EOL> benv . declare ( '<STR_LIT>' , getattr ( builtins , '<STR_LIT>' , getattr ( builtins , '<STR_LIT:str>' ) ) ) <EOL> benv . declare ( '<STR_LIT>' , getattr ( builtins , '<STR_LIT>' , getattr ( builtins , '<STR_LIT:int>' ) ) ) <EOL> benv . declare ( '<STR_LIT>' , getattr ( builtins , '<STR_LIT>' , getattr ( builtins , '<STR_LIT>' ) ) ) <EOL> denv = CompileTimeScope ( benv ) <EOL> return denv <EOL> class SourceDescriptor ( object ) : <EOL> """<STR_LIT>""" <EOL> _file_type = '<STR_LIT>' <EOL> _escaped_description = None <EOL> _cmp_name = '<STR_LIT>' <EOL> def __str__ ( self ) : <EOL> assert False <EOL> def set_file_type_from_name ( self , filename ) : <EOL> name , ext = os . path . splitext ( filename ) <EOL> self . _file_type = ext in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) and ext [ <NUM_LIT:1> : ] or '<STR_LIT>' <EOL> def is_cython_file ( self ) : <EOL> return self . _file_type in ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> def is_python_file ( self ) : <EOL> return self . _file_type == '<STR_LIT>' <EOL> def get_escaped_description ( self ) : <EOL> if self . _escaped_description is None : <EOL> esc_desc = self . get_description ( ) . encode ( '<STR_LIT>' , '<STR_LIT:replace>' ) . decode ( "<STR_LIT>" ) <EOL> self . _escaped_description = esc_desc . replace ( '<STR_LIT:\\>' , '<STR_LIT:/>' ) <EOL> return self . _escaped_description <EOL> def __gt__ ( self , other ) : <EOL> try : <EOL> return self . _cmp_name > other . _cmp_name <EOL> except AttributeError : <EOL> return False <EOL> def __lt__ ( self , other ) : <EOL> try : <EOL> return self . _cmp_name < other . _cmp_name <EOL> except AttributeError : <EOL> return False <EOL> def __le__ ( self , other ) : <EOL> try : <EOL> return self . _cmp_name <= other . _cmp_name <EOL> except AttributeError : <EOL> return False <EOL> class FileSourceDescriptor ( SourceDescriptor ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , filename , path_description = None ) : <EOL> filename = Utils . decode_filename ( filename ) <EOL> self . path_description = path_description or filename <EOL> self . filename = filename <EOL> self . set_file_type_from_name ( filename ) <EOL> self . _cmp_name = filename <EOL> self . _lines = { } <EOL> def get_lines ( self , encoding = None , error_handling = None ) : <EOL> key = ( encoding , error_handling ) <EOL> try : <EOL> lines = self . _lines [ key ] <EOL> if lines is not None : <EOL> return lines <EOL> except KeyError : <EOL> pass <EOL> with Utils . open_source_file ( self . filename , encoding = encoding , error_handling = error_handling ) as f : <EOL> lines = list ( f ) <EOL> if key in self . _lines : <EOL> self . _lines [ key ] = lines <EOL> else : <EOL> self . _lines [ key ] = None <EOL> return lines <EOL> def get_description ( self ) : <EOL> try : <EOL> return os . path . relpath ( self . path_description ) <EOL> except ValueError : <EOL> return self . path_description <EOL> def get_error_description ( self ) : <EOL> path = self . filename <EOL> cwd = Utils . decode_filename ( os . getcwd ( ) + os . path . sep ) <EOL> if path . startswith ( cwd ) : <EOL> return path [ len ( cwd ) : ] <EOL> return path <EOL> def get_filenametable_entry ( self ) : <EOL> return self . filename <EOL> def __eq__ ( self , other ) : <EOL> return isinstance ( other , FileSourceDescriptor ) and self . filename == other . filename <EOL> def __hash__ ( self ) : <EOL> return hash ( self . filename ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % self . filename <EOL> class StringSourceDescriptor ( SourceDescriptor ) : <EOL> """<STR_LIT>""" <EOL> filename = None <EOL> def __init__ ( self , name , code ) : <EOL> self . name = name <EOL> self . codelines = [ x + "<STR_LIT:\n>" for x in code . split ( "<STR_LIT:\n>" ) ] <EOL> self . _cmp_name = name <EOL> def get_lines ( self , encoding = None , error_handling = None ) : <EOL> if not encoding : <EOL> return self . codelines <EOL> else : <EOL> return [ line . encode ( encoding , error_handling ) . decode ( encoding ) <EOL> for line in self . codelines ] <EOL> def get_description ( self ) : <EOL> return self . name <EOL> get_error_description = get_description <EOL> def get_filenametable_entry ( self ) : <EOL> return "<STR_LIT>" <EOL> def __hash__ ( self ) : <EOL> return id ( self ) <EOL> def __eq__ ( self , other ) : <EOL> return isinstance ( other , StringSourceDescriptor ) and self . name == other . name <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % self . name <EOL> class PyrexScanner ( Scanner ) : <EOL> def __init__ ( self , file , filename , parent_scanner = None , <EOL> scope = None , context = None , source_encoding = None , parse_comments = True , initial_pos = None ) : <EOL> Scanner . __init__ ( self , get_lexicon ( ) , file , filename , initial_pos ) <EOL> if parent_scanner : <EOL> self . context = parent_scanner . context <EOL> self . included_files = parent_scanner . included_files <EOL> self . compile_time_env = parent_scanner . compile_time_env <EOL> self . compile_time_eval = parent_scanner . compile_time_eval <EOL> self . compile_time_expr = parent_scanner . compile_time_expr <EOL> else : <EOL> self . context = context <EOL> self . included_files = scope . included_files <EOL> self . compile_time_env = initial_compile_time_env ( ) <EOL> self . compile_time_eval = <NUM_LIT:1> <EOL> self . compile_time_expr = <NUM_LIT:0> <EOL> if getattr ( context . options , '<STR_LIT>' , None ) : <EOL> self . compile_time_env . update ( context . options . compile_time_env ) <EOL> self . parse_comments = parse_comments <EOL> self . source_encoding = source_encoding <EOL> if filename . is_python_file ( ) : <EOL> self . in_python_file = True <EOL> self . keywords = set ( py_reserved_words ) <EOL> else : <EOL> self . in_python_file = False <EOL> self . keywords = set ( pyx_reserved_words ) <EOL> self . trace = trace_scanner <EOL> self . indentation_stack = [ <NUM_LIT:0> ] <EOL> self . indentation_char = None <EOL> self . bracket_nesting_level = <NUM_LIT:0> <EOL> self . async_enabled = <NUM_LIT:0> <EOL> self . begin ( '<STR_LIT>' ) <EOL> self . sy = '<STR_LIT>' <EOL> self . next ( ) <EOL> def commentline ( self , text ) : <EOL> if self . parse_comments : <EOL> self . produce ( '<STR_LIT>' , text ) <EOL> def strip_underscores ( self , text , symbol ) : <EOL> self . produce ( symbol , text . replace ( '<STR_LIT:_>' , '<STR_LIT>' ) ) <EOL> def current_level ( self ) : <EOL> return self . indentation_stack [ - <NUM_LIT:1> ] <EOL> def open_bracket_action ( self , text ) : <EOL> self . bracket_nesting_level += <NUM_LIT:1> <EOL> return text <EOL> def close_bracket_action ( self , text ) : <EOL> self . bracket_nesting_level -= <NUM_LIT:1> <EOL> return text <EOL> def newline_action ( self , text ) : <EOL> if self . bracket_nesting_level == <NUM_LIT:0> : <EOL> self . begin ( '<STR_LIT>' ) <EOL> self . produce ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> string_states = { <EOL> "<STR_LIT:'>" : '<STR_LIT>' , <EOL> '<STR_LIT:">' : '<STR_LIT>' , <EOL> "<STR_LIT>" : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> } <EOL> def begin_string_action ( self , text ) : <EOL> while text [ : <NUM_LIT:1> ] in any_string_prefix : <EOL> text = text [ <NUM_LIT:1> : ] <EOL> self . begin ( self . string_states [ text ] ) <EOL> self . produce ( '<STR_LIT>' ) <EOL> def end_string_action ( self , text ) : <EOL> self . begin ( '<STR_LIT>' ) <EOL> self . produce ( '<STR_LIT>' ) <EOL> def unclosed_string_action ( self , text ) : <EOL> self . end_string_action ( text ) <EOL> self . error ( "<STR_LIT>" ) <EOL> def indentation_action ( self , text ) : <EOL> self . begin ( '<STR_LIT>' ) <EOL> if text : <EOL> c = text [ <NUM_LIT:0> ] <EOL> if self . indentation_char is None : <EOL> self . indentation_char = c <EOL> else : <EOL> if self . indentation_char != c : <EOL> self . error ( "<STR_LIT>" ) <EOL> if text . replace ( c , "<STR_LIT>" ) != "<STR_LIT>" : <EOL> self . error ( "<STR_LIT>" ) <EOL> current_level = self . current_level ( ) <EOL> new_level = len ( text ) <EOL> if new_level == current_level : <EOL> return <EOL> elif new_level > current_level : <EOL> self . indentation_stack . append ( new_level ) <EOL> self . produce ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> else : <EOL> while new_level < self . current_level ( ) : <EOL> self . indentation_stack . pop ( ) <EOL> self . produce ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if new_level != self . current_level ( ) : <EOL> self . error ( "<STR_LIT>" ) <EOL> def eof_action ( self , text ) : <EOL> while len ( self . indentation_stack ) > <NUM_LIT:1> : <EOL> self . produce ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . indentation_stack . pop ( ) <EOL> self . produce ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> def next ( self ) : <EOL> try : <EOL> sy , systring = self . read ( ) <EOL> except UnrecognizedInput : <EOL> self . error ( "<STR_LIT>" ) <EOL> return <EOL> if sy == IDENT : <EOL> if systring in self . keywords : <EOL> if systring == u'<STR_LIT>' and print_function in self . context . future_directives : <EOL> self . keywords . discard ( '<STR_LIT>' ) <EOL> elif systring == u'<STR_LIT>' and self . context . language_level >= <NUM_LIT:3> : <EOL> self . keywords . discard ( '<STR_LIT>' ) <EOL> else : <EOL> sy = systring <EOL> systring = self . context . intern_ustring ( systring ) <EOL> self . sy = sy <EOL> self . systring = systring <EOL> if False : <EOL> _ , line , col = self . position ( ) <EOL> if not self . systring or self . sy == self . systring : <EOL> t = self . sy <EOL> else : <EOL> t = "<STR_LIT>" % ( self . sy , self . systring ) <EOL> print ( "<STR_LIT>" % ( line , col , t ) ) <EOL> def peek ( self ) : <EOL> saved = self . sy , self . systring <EOL> self . next ( ) <EOL> next = self . sy , self . systring <EOL> self . unread ( * next ) <EOL> self . sy , self . systring = saved <EOL> return next <EOL> def put_back ( self , sy , systring ) : <EOL> self . unread ( self . sy , self . systring ) <EOL> self . sy = sy <EOL> self . systring = systring <EOL> def unread ( self , token , value ) : <EOL> self . queue . insert ( <NUM_LIT:0> , ( token , value ) ) <EOL> def error ( self , message , pos = None , fatal = True ) : <EOL> if pos is None : <EOL> pos = self . position ( ) <EOL> if self . sy == '<STR_LIT>' : <EOL> error ( pos , "<STR_LIT>" ) <EOL> err = error ( pos , message ) <EOL> if fatal : raise err <EOL> def expect ( self , what , message = None ) : <EOL> if self . sy == what : <EOL> self . next ( ) <EOL> else : <EOL> self . expected ( what , message ) <EOL> def expect_keyword ( self , what , message = None ) : <EOL> if self . sy == IDENT and self . systring == what : <EOL> self . next ( ) <EOL> else : <EOL> self . expected ( what , message ) <EOL> def expected ( self , what , message = None ) : <EOL> if message : <EOL> self . error ( message ) <EOL> else : <EOL> if self . sy == IDENT : <EOL> found = self . systring <EOL> else : <EOL> found = self . sy <EOL> self . error ( "<STR_LIT>" % ( what , found ) ) <EOL> def expect_indent ( self ) : <EOL> self . expect ( '<STR_LIT>' , "<STR_LIT>" ) <EOL> def expect_dedent ( self ) : <EOL> self . expect ( '<STR_LIT>' , "<STR_LIT>" ) <EOL> def expect_newline ( self , message = "<STR_LIT>" , ignore_semicolon = False ) : <EOL> useless_trailing_semicolon = None <EOL> if ignore_semicolon and self . sy == '<STR_LIT:;>' : <EOL> useless_trailing_semicolon = self . position ( ) <EOL> self . next ( ) <EOL> if self . sy != '<STR_LIT>' : <EOL> self . expect ( '<STR_LIT>' , message ) <EOL> if useless_trailing_semicolon is not None : <EOL> warning ( useless_trailing_semicolon , "<STR_LIT>" ) <EOL> def enter_async ( self ) : <EOL> self . async_enabled += <NUM_LIT:1> <EOL> if self . async_enabled == <NUM_LIT:1> : <EOL> self . keywords . add ( '<STR_LIT>' ) <EOL> self . keywords . add ( '<STR_LIT>' ) <EOL> def exit_async ( self ) : <EOL> assert self . async_enabled > <NUM_LIT:0> <EOL> self . async_enabled -= <NUM_LIT:1> <EOL> if not self . async_enabled : <EOL> self . keywords . discard ( '<STR_LIT>' ) <EOL> self . keywords . discard ( '<STR_LIT>' ) <EOL> if self . sy in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> self . sy , self . systring = IDENT , self . context . intern_ustring ( self . sy ) </s>
