<s> """<STR_LIT>""" <EOL> import re <EOL> from itertools import chain <EOL> from jedi . _compatibility import use_metaclass <EOL> from jedi import settings <EOL> from jedi . parser import Parser <EOL> from jedi . parser import tree <EOL> from jedi import cache <EOL> from jedi import debug <EOL> from jedi . parser . tokenize import ( source_tokens , NEWLINE , <EOL> ENDMARKER , INDENT , DEDENT ) <EOL> FLOWS = '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' <EOL> class FastModule ( tree . Module ) : <EOL> type = '<STR_LIT>' <EOL> def __init__ ( self , module_path ) : <EOL> super ( FastModule , self ) . __init__ ( [ ] ) <EOL> self . modules = [ ] <EOL> self . reset_caches ( ) <EOL> self . names_dict = { } <EOL> self . path = module_path <EOL> def reset_caches ( self ) : <EOL> self . modules = [ ] <EOL> try : <EOL> del self . _used_names <EOL> except AttributeError : <EOL> pass <EOL> @ property <EOL> @ cache . underscore_memoization <EOL> def used_names ( self ) : <EOL> return MergedNamesDict ( [ m . used_names for m in self . modules ] ) <EOL> @ property <EOL> def global_names ( self ) : <EOL> return [ name for m in self . modules for name in m . global_names ] <EOL> @ property <EOL> def error_statement_stacks ( self ) : <EOL> return [ e for m in self . modules for e in m . error_statement_stacks ] <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( type ( self ) . __name__ , self . name , <EOL> self . start_pos [ <NUM_LIT:0> ] , self . end_pos [ <NUM_LIT:0> ] ) <EOL> @ global_names . setter <EOL> def global_names ( self , value ) : <EOL> pass <EOL> @ error_statement_stacks . setter <EOL> def error_statement_stacks ( self , value ) : <EOL> pass <EOL> @ used_names . setter <EOL> def used_names ( self , value ) : <EOL> pass <EOL> class MergedNamesDict ( object ) : <EOL> def __init__ ( self , dicts ) : <EOL> self . dicts = dicts <EOL> def __iter__ ( self ) : <EOL> return iter ( set ( key for dct in self . dicts for key in dct ) ) <EOL> def __getitem__ ( self , value ) : <EOL> return list ( chain . from_iterable ( dct . get ( value , [ ] ) for dct in self . dicts ) ) <EOL> def items ( self ) : <EOL> dct = { } <EOL> for d in self . dicts : <EOL> for key , values in d . items ( ) : <EOL> try : <EOL> dct_values = dct [ key ] <EOL> dct_values += values <EOL> except KeyError : <EOL> dct [ key ] = list ( values ) <EOL> return dct . items ( ) <EOL> def values ( self ) : <EOL> lst = [ ] <EOL> for dct in self . dicts : <EOL> lst += dct . values ( ) <EOL> return lst <EOL> class CachedFastParser ( type ) : <EOL> """<STR_LIT>""" <EOL> def __call__ ( self , grammar , source , module_path = None ) : <EOL> if not settings . fast_parser : <EOL> return Parser ( grammar , source , module_path ) <EOL> pi = cache . parser_cache . get ( module_path , None ) <EOL> if pi is None or isinstance ( pi . parser , Parser ) : <EOL> p = super ( CachedFastParser , self ) . __call__ ( grammar , source , module_path ) <EOL> else : <EOL> p = pi . parser <EOL> p . update ( source ) <EOL> return p <EOL> class ParserNode ( object ) : <EOL> def __init__ ( self , fast_module , parser , source ) : <EOL> self . _fast_module = fast_module <EOL> self . parent = None <EOL> self . _node_children = [ ] <EOL> self . source = source <EOL> self . hash = hash ( source ) <EOL> self . parser = parser <EOL> try : <EOL> self . _content_scope = parser . module . subscopes [ <NUM_LIT:0> ] <EOL> except IndexError : <EOL> self . _content_scope = parser . module <EOL> else : <EOL> self . _rewrite_last_newline ( ) <EOL> self . _old_children = list ( self . _content_scope . children ) <EOL> def _rewrite_last_newline ( self ) : <EOL> """<STR_LIT>""" <EOL> c = self . _content_scope . children <EOL> if tree . is_node ( c [ - <NUM_LIT:1> ] , '<STR_LIT>' ) : <EOL> end_marker = self . parser . module . children [ - <NUM_LIT:1> ] <EOL> c [ - <NUM_LIT:1> ] . children [ - <NUM_LIT:1> ] . prefix = end_marker . prefix <EOL> end_marker . prefix = '<STR_LIT>' <EOL> def __repr__ ( self ) : <EOL> module = self . parser . module <EOL> try : <EOL> return '<STR_LIT>' % ( type ( self ) . __name__ , module . start_pos , module . end_pos ) <EOL> except IndexError : <EOL> return '<STR_LIT>' % type ( self ) . __name__ <EOL> def reset_node ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _node_children = [ ] <EOL> scope = self . _content_scope <EOL> scope . children = list ( self . _old_children ) <EOL> try : <EOL> scope . names_dict = scope . names_dict . dicts [ <NUM_LIT:0> ] <EOL> except AttributeError : <EOL> pass <EOL> def close ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _node_children : <EOL> dcts = [ n . parser . module . names_dict for n in self . _node_children ] <EOL> dcts . insert ( <NUM_LIT:0> , self . _content_scope . names_dict ) <EOL> self . _content_scope . names_dict = MergedNamesDict ( dcts ) <EOL> def parent_until_indent ( self , indent = None ) : <EOL> if ( indent is None or self . _indent >= indent ) and self . parent is not None : <EOL> self . close ( ) <EOL> return self . parent . parent_until_indent ( indent ) <EOL> return self <EOL> @ property <EOL> def _indent ( self ) : <EOL> if not self . parent : <EOL> return <NUM_LIT:0> <EOL> return self . parser . module . children [ <NUM_LIT:0> ] . start_pos [ <NUM_LIT:1> ] <EOL> def add_node ( self , node , line_offset ) : <EOL> """<STR_LIT>""" <EOL> m = node . parser . module <EOL> node . parser . position_modifier . line = line_offset <EOL> self . _fast_module . modules . append ( m ) <EOL> node . parent = self <EOL> self . _node_children . append ( node ) <EOL> scope = self . _content_scope <EOL> for child in m . children : <EOL> child . parent = scope <EOL> scope . children . append ( child ) <EOL> return node <EOL> def all_sub_nodes ( self ) : <EOL> """<STR_LIT>""" <EOL> for n in self . _node_children : <EOL> yield n <EOL> for y in n . all_sub_nodes ( ) : <EOL> yield y <EOL> @ cache . underscore_memoization <EOL> def remove_last_newline ( self ) : <EOL> self . parser . remove_last_newline ( ) <EOL> class FastParser ( use_metaclass ( CachedFastParser ) ) : <EOL> _FLOWS_NEED_SPACE = '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' <EOL> _FLOWS_NEED_COLON = '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' <EOL> _keyword_re = re . compile ( '<STR_LIT>' <EOL> % ( '<STR_LIT:|>' . join ( _FLOWS_NEED_SPACE ) , <EOL> '<STR_LIT:|>' . join ( _FLOWS_NEED_COLON ) ) ) <EOL> def __init__ ( self , grammar , source , module_path = None ) : <EOL> self . _grammar = grammar <EOL> self . module_path = module_path <EOL> self . _reset_caches ( ) <EOL> self . update ( source ) <EOL> def _reset_caches ( self ) : <EOL> self . module = FastModule ( self . module_path ) <EOL> self . current_node = ParserNode ( self . module , self , '<STR_LIT>' ) <EOL> def update ( self , source ) : <EOL> self . number_parsers_used = <NUM_LIT:0> <EOL> self . number_of_splits = <NUM_LIT:0> <EOL> self . number_of_misses = <NUM_LIT:0> <EOL> self . module . reset_caches ( ) <EOL> try : <EOL> self . _parse ( source ) <EOL> except : <EOL> self . _reset_caches ( ) <EOL> raise <EOL> def _split_parts ( self , source ) : <EOL> """<STR_LIT>""" <EOL> def gen_part ( ) : <EOL> text = '<STR_LIT>' . join ( current_lines ) <EOL> del current_lines [ : ] <EOL> self . number_of_splits += <NUM_LIT:1> <EOL> return text <EOL> def just_newlines ( current_lines ) : <EOL> for line in current_lines : <EOL> line = line . lstrip ( '<STR_LIT>' ) <EOL> if line and line [ <NUM_LIT:0> ] != '<STR_LIT:#>' : <EOL> return False <EOL> return True <EOL> self . _lines = source . splitlines ( True ) <EOL> current_lines = [ ] <EOL> is_decorator = False <EOL> indent_list = [ - <NUM_LIT:1> , <NUM_LIT:0> ] <EOL> new_indent = False <EOL> parentheses_level = <NUM_LIT:0> <EOL> flow_indent = None <EOL> previous_line = None <EOL> for i , l in enumerate ( self . _lines ) : <EOL> if l . endswith ( '<STR_LIT>' ) or l . endswith ( '<STR_LIT>' ) : <EOL> if previous_line is not None : <EOL> previous_line += l <EOL> else : <EOL> previous_line = l <EOL> continue <EOL> if previous_line is not None : <EOL> l = previous_line + l <EOL> previous_line = None <EOL> s = l . lstrip ( '<STR_LIT>' ) <EOL> indent = len ( l ) - len ( s ) <EOL> if not s or s [ <NUM_LIT:0> ] == '<STR_LIT:#>' : <EOL> current_lines . append ( l ) <EOL> continue <EOL> if new_indent : <EOL> if indent > indent_list [ - <NUM_LIT:2> ] : <EOL> indent_list [ - <NUM_LIT:1> ] = indent <EOL> new_indent = False <EOL> while indent <= indent_list [ - <NUM_LIT:2> ] : <EOL> indent_list . pop ( ) <EOL> new_indent = False <EOL> if flow_indent is None and current_lines and not parentheses_level : <EOL> yield gen_part ( ) <EOL> flow_indent = None <EOL> if flow_indent is None : <EOL> m = self . _keyword_re . match ( l ) <EOL> if m : <EOL> if m . group ( <NUM_LIT:1> ) . strip ( '<STR_LIT>' ) in FLOWS : <EOL> if not parentheses_level : <EOL> flow_indent = indent <EOL> else : <EOL> if not is_decorator and not just_newlines ( current_lines ) : <EOL> yield gen_part ( ) <EOL> is_decorator = '<STR_LIT:@>' == m . group ( <NUM_LIT:1> ) <EOL> if not is_decorator : <EOL> parentheses_level = <NUM_LIT:0> <EOL> indent_list . append ( indent + <NUM_LIT:1> ) <EOL> new_indent = True <EOL> elif is_decorator : <EOL> is_decorator = False <EOL> parentheses_level = max ( <NUM_LIT:0> , ( l . count ( '<STR_LIT:(>' ) + l . count ( '<STR_LIT:[>' ) + l . count ( '<STR_LIT:{>' ) <EOL> - l . count ( '<STR_LIT:)>' ) - l . count ( '<STR_LIT:]>' ) - l . count ( '<STR_LIT:}>' ) ) ) <EOL> current_lines . append ( l ) <EOL> if current_lines : <EOL> yield gen_part ( ) <EOL> def _parse ( self , source ) : <EOL> """<STR_LIT>""" <EOL> added_newline = False <EOL> if not source or source [ - <NUM_LIT:1> ] != '<STR_LIT:\n>' : <EOL> source += '<STR_LIT:\n>' <EOL> added_newline = True <EOL> next_line_offset = line_offset = <NUM_LIT:0> <EOL> start = <NUM_LIT:0> <EOL> nodes = list ( self . current_node . all_sub_nodes ( ) ) <EOL> self . current_node . reset_node ( ) <EOL> last_end_line = <NUM_LIT:1> <EOL> for code_part in self . _split_parts ( source ) : <EOL> next_line_offset += code_part . count ( '<STR_LIT:\n>' ) <EOL> if line_offset + <NUM_LIT:1> == last_end_line : <EOL> self . current_node = self . _get_node ( code_part , source [ start : ] , <EOL> line_offset , nodes ) <EOL> else : <EOL> self . number_of_misses += <NUM_LIT:1> <EOL> while last_end_line < next_line_offset + <NUM_LIT:1> : <EOL> line_offset = last_end_line - <NUM_LIT:1> <EOL> src = '<STR_LIT>' . join ( self . _lines [ line_offset : ] ) <EOL> self . current_node = self . _get_node ( code_part , src , <EOL> line_offset , nodes ) <EOL> last_end_line = self . current_node . parser . module . end_pos [ <NUM_LIT:0> ] <EOL> debug . dbg ( '<STR_LIT>' , <EOL> self . module_path , line_offset + <NUM_LIT:1> ) <EOL> line_offset = next_line_offset <EOL> start += len ( code_part ) <EOL> last_end_line = self . current_node . parser . module . end_pos [ <NUM_LIT:0> ] <EOL> if added_newline : <EOL> self . current_node . remove_last_newline ( ) <EOL> self . current_node = self . current_node . parent_until_indent ( ) <EOL> self . current_node . close ( ) <EOL> debug . dbg ( '<STR_LIT>' <EOL> % ( self . module_path , self . number_parsers_used , <EOL> self . number_of_splits ) ) <EOL> def _get_node ( self , source , parser_code , line_offset , nodes ) : <EOL> """<STR_LIT>""" <EOL> indent = len ( source ) - len ( source . lstrip ( '<STR_LIT>' ) ) <EOL> self . current_node = self . current_node . parent_until_indent ( indent ) <EOL> h = hash ( source ) <EOL> for index , node in enumerate ( nodes ) : <EOL> if node . hash == h and node . source == source : <EOL> node . reset_node ( ) <EOL> nodes . remove ( node ) <EOL> break <EOL> else : <EOL> tokenizer = FastTokenizer ( parser_code ) <EOL> self . number_parsers_used += <NUM_LIT:1> <EOL> p = Parser ( self . _grammar , parser_code , self . module_path , tokenizer = tokenizer ) <EOL> end = line_offset + p . module . end_pos [ <NUM_LIT:0> ] <EOL> used_lines = self . _lines [ line_offset : end - <NUM_LIT:1> ] <EOL> code_part_actually_used = '<STR_LIT>' . join ( used_lines ) <EOL> node = ParserNode ( self . module , p , code_part_actually_used ) <EOL> self . current_node . add_node ( node , line_offset ) <EOL> return node <EOL> class FastTokenizer ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , source ) : <EOL> self . source = source <EOL> self . _gen = source_tokens ( source ) <EOL> self . _closed = False <EOL> self . current = self . previous = NEWLINE , '<STR_LIT>' , ( <NUM_LIT:0> , <NUM_LIT:0> ) <EOL> self . _in_flow = False <EOL> self . _is_decorator = False <EOL> self . _first_stmt = True <EOL> self . _parentheses_level = <NUM_LIT:0> <EOL> self . _indent_counter = <NUM_LIT:0> <EOL> self . _flow_indent_counter = <NUM_LIT:0> <EOL> self . _returned_endmarker = False <EOL> self . _expect_indent = False <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def next ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . __next__ ( ) <EOL> def __next__ ( self ) : <EOL> if self . _closed : <EOL> return self . _finish_dedents ( ) <EOL> typ , value , start_pos , prefix = current = next ( self . _gen ) <EOL> if typ == ENDMARKER : <EOL> self . _closed = True <EOL> self . _returned_endmarker = True <EOL> return current <EOL> self . previous = self . current <EOL> self . current = current <EOL> if typ == INDENT : <EOL> self . _indent_counter += <NUM_LIT:1> <EOL> if not self . _expect_indent and not self . _first_stmt and not self . _in_flow : <EOL> self . _flow_indent_counter = self . _indent_counter - <NUM_LIT:1> <EOL> self . _in_flow = True <EOL> self . _expect_indent = False <EOL> elif typ == DEDENT : <EOL> self . _indent_counter -= <NUM_LIT:1> <EOL> if self . _in_flow : <EOL> if self . _indent_counter == self . _flow_indent_counter : <EOL> self . _in_flow = False <EOL> else : <EOL> self . _closed = True <EOL> return current <EOL> if value in ( '<STR_LIT>' , '<STR_LIT:class>' ) and self . _parentheses_level and re . search ( r'<STR_LIT>' , prefix ) : <EOL> self . _parentheses_level = <NUM_LIT:0> <EOL> return self . _close ( ) <EOL> if self . previous [ <NUM_LIT:0> ] in ( NEWLINE , INDENT , DEDENT ) and not self . _parentheses_level and typ not in ( INDENT , DEDENT ) : <EOL> if not self . _in_flow : <EOL> if value in FLOWS : <EOL> self . _flow_indent_counter = self . _indent_counter <EOL> self . _first_stmt = False <EOL> elif value in ( '<STR_LIT>' , '<STR_LIT:class>' , '<STR_LIT:@>' ) : <EOL> if not self . _first_stmt and not self . _is_decorator : <EOL> return self . _close ( ) <EOL> self . _is_decorator = '<STR_LIT:@>' == value <EOL> if not self . _is_decorator : <EOL> self . _first_stmt = False <EOL> self . _expect_indent = True <EOL> elif self . _expect_indent : <EOL> return self . _close ( ) <EOL> else : <EOL> self . _first_stmt = False <EOL> if value in '<STR_LIT>' and value : <EOL> self . _parentheses_level += <NUM_LIT:1> <EOL> elif value in '<STR_LIT>' and value : <EOL> self . _parentheses_level = max ( self . _parentheses_level - <NUM_LIT:1> , <NUM_LIT:0> ) <EOL> return current <EOL> def _close ( self ) : <EOL> if self . _first_stmt : <EOL> if self . current [ <NUM_LIT:1> ] != '<STR_LIT:@>' : <EOL> self . _first_stmt = False <EOL> return self . current <EOL> else : <EOL> self . _closed = True <EOL> return self . _finish_dedents ( ) <EOL> def _finish_dedents ( self ) : <EOL> if self . _indent_counter : <EOL> self . _indent_counter -= <NUM_LIT:1> <EOL> return DEDENT , '<STR_LIT>' , self . current [ <NUM_LIT:2> ] , '<STR_LIT>' <EOL> elif not self . _returned_endmarker : <EOL> self . _returned_endmarker = True <EOL> return ENDMARKER , '<STR_LIT>' , self . current [ <NUM_LIT:2> ] , self . _get_prefix ( ) <EOL> else : <EOL> raise StopIteration <EOL> def _get_prefix ( self ) : <EOL> """<STR_LIT>""" <EOL> cur = self . current <EOL> while cur [ <NUM_LIT:0> ] == DEDENT : <EOL> cur = next ( self . _gen ) <EOL> prefix = cur [ <NUM_LIT:3> ] <EOL> return re . sub ( r'<STR_LIT>' , '<STR_LIT>' , prefix ) </s>
